private async Task CheckClusterManifestVersion(string clusterManifestVersion, CancellationToken cancellationToken) { // If an upgrade is still in progress, don't do anything FabricUpgradeProgress upgradeProgress = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <FabricUpgradeProgress>( () => this.fabricClient.ClusterManager.GetFabricUpgradeProgressAsync(FabricClient.DefaultTimeout, cancellationToken)).ConfigureAwait(false); if (upgradeProgress == null || !FabricClientWrapper.IsUpgradeCompleted(upgradeProgress.UpgradeState)) { return; } string clusterManifestContent = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( () => this.fabricClient.ClusterManager.GetClusterManifestAsync(FabricClient.DefaultTimeout, cancellationToken)).ConfigureAwait(false); var clusterManifest = XMLHelper.ReadClusterManifestFromContent(clusterManifestContent); var actualManifestVersion = clusterManifest.Version; if (!actualManifestVersion.Equals(clusterManifestVersion, StringComparison.OrdinalIgnoreCase)) { this.fabricClient.HealthManager.ReportHealth(new ClusterHealthReport(new HealthInformation( Constants.UpgradeOrchestrationHealthSourceId, Constants.ManifestVersionMismatchHealthProperty, HealthState.Warning) { Description = string.Format( StringResources.WarningHealth_ClusterManifestVersionMismatch, actualManifestVersion, clusterManifestVersion), TimeToLive = TimeSpan.FromHours(48), RemoveWhenExpired = true })); } else { this.fabricClient.HealthManager.ReportHealth(new ClusterHealthReport(new HealthInformation( Constants.UpgradeOrchestrationHealthSourceId, Constants.ManifestVersionMismatchHealthProperty, HealthState.Ok) { TimeToLive = TimeSpan.FromMilliseconds(1), RemoveWhenExpired = true })); } }
private async Task PollGoalStateForCodePackagesAsync(CancellationToken cancellationToken) { var fabricClient = new FabricClient(); //// region: Initialization Parameters FabricUpgradeProgress upgradeProgress; try { upgradeProgress = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <FabricUpgradeProgress>( () => fabricClient.ClusterManager.GetFabricUpgradeProgressAsync(TimeSpan.FromMinutes(DMConstants.FabricOperationTimeoutInMinutes), cancellationToken), TimeSpan.FromMinutes(DMConstants.FabricOperationTimeoutInMinutes), cancellationToken).ConfigureAwait(false); } catch (Exception ex) { this.traceSource.WriteError(TraceType, "GetFabricUpgradeProgressAsync threw: {0}", ex.ToString()); this.timerInterval = TimeSpan.FromMinutes(DMConstants.FabricUpgradeStatePollIntervalInMinutes); return; } if (!FabricClientWrapper.IsUpgradeCompleted(upgradeProgress.UpgradeState)) { this.timerInterval = TimeSpan.FromMinutes(DMConstants.FabricUpgradeStatePollIntervalInMinutes); this.traceSource.WriteInfo(TraceType, "Cannot retrieve cluster version; FabricUpgradeState is currently: {0}. Will retry in {1}.", upgradeProgress.UpgradeState.ToString(), this.timerInterval.ToString()); return; } string currentVersionStr = upgradeProgress.TargetCodeVersion; Version currentVersion = Version.Parse(currentVersionStr); string packageDropDir = System.Fabric.Common.Helpers.GetNewTempPath(); NativeConfigStore configStore = NativeConfigStore.FabricGetConfigStore(); string goalStateUriStr = configStore.ReadUnencryptedString(DMConstants.UpgradeOrchestrationServiceConfigSectionName, DMConstants.GoalStateFileUrlName); if (string.IsNullOrEmpty(goalStateUriStr)) { goalStateUriStr = DMConstants.DefaultGoalStateFileUrl; } //// endregion this.traceSource.WriteInfo(TraceType, "PollCodePackagesFromGoalState currentVersion: {0}, packageDropDir: {1} goalStateUri: {2}", currentVersionStr, packageDropDir, goalStateUriStr); try { Uri goalStateUri = null; string goalStateJson = null; if (!Uri.TryCreate(goalStateUriStr, UriKind.Absolute, out goalStateUri)) { string errorMessage = string.Format("Cannot parse GoalStateUri: {0}", goalStateUriStr); this.traceSource.WriteError(TraceType, errorMessage); ReleaseAssert.Fail(errorMessage); return; } if (!(await StandaloneUtility.IsUriReachableAsync(goalStateUri).ConfigureAwait(false))) { this.timerInterval = TimeSpan.FromMinutes(DMConstants.FabricGoalStateReachablePollIntervalInMinutes); this.traceSource.WriteWarning(TraceType, "Cannot reach download uri for goal state file: {0}. Will retry in {1}.", goalStateUri.AbsoluteUri, this.timerInterval.ToString()); this.EmitGoalStateReachableHealth(fabricClient, false); return; } else { this.EmitGoalStateReachableHealth(fabricClient, true); } goalStateJson = await GoalStateModel.GetGoalStateFileJsonAsync(goalStateUri).ConfigureAwait(false); if (string.IsNullOrEmpty(goalStateJson)) { this.traceSource.WriteWarning(TraceType, "Loaded goal state JSON is empty."); return; } GoalStateModel model = GoalStateModel.GetGoalStateModelFromJson(goalStateJson); if (model == null || model.Packages == null) { this.traceSource.WriteWarning(TraceType, "Goal state JSON could not be deserialized:\n{0}", goalStateJson); return; } string availableGoalStateVersions = string.Format("Available versions in goal state file: {0}", model.ToString()); this.traceSource.WriteInfo(TraceType, availableGoalStateVersions); IEnumerable <PackageDetails> candidatePackages = model.Packages.Where( package => currentVersion < Version.Parse(package.Version)); if (candidatePackages.Count() == 0) { this.traceSource.WriteInfo(TraceType, "No upgrades available."); } string versionCurrentStr = currentVersionStr; Version versionCurrent = currentVersion; PackageDetails targetPackage = null; for (IEnumerable <PackageDetails> availableCandidatePackages = candidatePackages.Where( package => (package.MinVersion == null || Version.Parse(package.MinVersion) <= versionCurrent)); availableCandidatePackages.Count() > 0; versionCurrentStr = targetPackage.Version, versionCurrent = Version.Parse(versionCurrentStr), availableCandidatePackages = candidatePackages.Where( package => (versionCurrent < Version.Parse(package.Version) && (package.MinVersion == null || Version.Parse(package.MinVersion) <= versionCurrent)))) { if (!IsVersionUpgradeable(versionCurrentStr, model)) { this.traceSource.WriteInfo(TraceType, "Version {0} is not upgradeable.", versionCurrentStr); break; } int numPackages = availableCandidatePackages.Count(); if (numPackages == 0) { this.traceSource.WriteInfo(TraceType, "No upgrade available."); return; } else if (numPackages == 1) { targetPackage = availableCandidatePackages.First(); } else { Version maxVersion = StandaloneGoalStateProvisioner.ZeroVersion; foreach (PackageDetails package in availableCandidatePackages) { Version targetVersion; if (!Version.TryParse(package.Version, out targetVersion)) { this.traceSource.WriteWarning(TraceType, "Package {0} version could not be parsed. Trying another one.", package.Version); continue; } if (targetVersion > maxVersion) { targetPackage = package; maxVersion = targetVersion; } } } try { if (await this.IsPackageProvisionedAsync(targetPackage.Version, fabricClient, cancellationToken).ConfigureAwait(false)) { this.traceSource.WriteInfo(TraceType, "Package {0} is already provisioned.", targetPackage.Version); continue; } } catch (Exception ex) { this.traceSource.WriteError(TraceType, "PackageIsProvisioned for {0} threw: {1}", targetPackage.Version, ex.ToString()); } string packageLocalDownloadPath = Path.Combine(packageDropDir, string.Format(DMConstants.SFPackageDropNameFormat, targetPackage.Version)); if (await StandaloneUtility.DownloadPackageAsync(targetPackage.Version, targetPackage.TargetPackageLocation, packageLocalDownloadPath).ConfigureAwait(false)) { try { this.traceSource.WriteInfo(TraceType, "Copying and provisioning version {0} from {1}.", targetPackage.Version, packageLocalDownloadPath); await ProvisionPackageAsync(targetPackage.Version, packageLocalDownloadPath, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { this.traceSource.WriteError(TraceType, "ProvisionPackageAsync for {0}, package {1} threw: {2}", targetPackage.Version, packageLocalDownloadPath, ex.ToString()); } } } // Determine if current version is latest/doesn't have expiry date. If package expiring, post cluster health warning. PackageDetails currentPackage = model.Packages.Where(package => package.Version == currentVersionStr).FirstOrDefault(); if (currentPackage != null && currentPackage.SupportExpiryDate != null && !currentPackage.IsGoalPackage && !this.IsAutoupgradeInstallEnabled() && this.IsPackageNearExpiry(currentPackage)) { this.traceSource.WriteWarning(TraceType, "Current version {0} expires {1}; emitting cluster warning.", currentVersionStr, currentPackage.SupportExpiryDate); this.EmitClusterVersionSupportedHealth(fabricClient, false, currentVersionStr, currentPackage.SupportExpiryDate); } else { this.traceSource.WriteInfo(TraceType, "Current version {0} is supported. Cluster health OK.", currentVersionStr); this.EmitClusterVersionSupportedHealth(fabricClient, true, null, null); } if (targetPackage != null && !string.IsNullOrEmpty(targetPackage.Version) && this.IsAutoupgradeInstallEnabled()) { try { // fire off an upgrade. this.StartCodeUpgrade(targetPackage.Version, fabricClient); } catch (Exception ex) { this.traceSource.WriteError(TraceType, "StartCodeUpgrade for version {0} threw: {1}", targetPackage.Version, ex.ToString()); } } } finally { if (Directory.Exists(packageDropDir)) { try { FabricDirectory.Delete(packageDropDir, true, true); } catch { } } } }