private static bool CheckNoMachineIsDomainController(MachineHealthContainer machineHealthContainer)
{
Parallel.ForEach(
machineHealthContainer.GetHealthyMachineNames(),
(string machineName) =>
{
bool result = true;
try
{
if (StandaloneUtility.IsMachineDomainController(machineName))
{
SFDeployerTrace.WriteError(StringResources.Error_BPAMachineIsDomainController, machineName);
result = false;
}
}
catch (System.ComponentModel.Win32Exception ex)
{
SFDeployerTrace.WriteError(StringResources.Error_BPADomainControllerQueryException, machineName, ex.NativeErrorCode, ex.Message);
result = false;
}
if (!result)
{
machineHealthContainer.MarkMachineAsUnhealthy(machineName);
}
});
return(machineHealthContainer.EnoughHealthyMachines());
}
// Clean install requires machine be free of previous Fabric installations
private static bool CheckForCleanInstall(StandAloneInstallerJsonModelBase config, MachineHealthContainer machineHealthContainer, bool isForcedRun = false)
{
SFDeployerTrace.WriteNoise(StringResources.Info_BPANoFabric);
List <string> machineNamesTemp = StandaloneUtility.GetMachineNamesIncludingClient(machineHealthContainer.GetHealthyMachineNames());
var importantSettings = config.GetFabricSystemSettings();
string fabricDataRoot = importantSettings.ContainsKey(DMConstants.FabricDataRootString) ?
importantSettings[DMConstants.FabricDataRootString] :
null;
bool localMachineFailed = false;
Parallel.ForEach(
machineNamesTemp,
(string machineName) =>
{
bool result = true;
if (StandaloneUtility.IsFabricInstalled(machineName))
{
SFDeployerTrace.WriteError(StringResources.Error_BPAPreviousFabricExists, machineName);
result = false;
}
if (!isForcedRun)
{
if (fabricDataRoot != null)
{
IEnumerable <string> machineNodes = config.Nodes.Where(n => n.IPAddress == machineName).Select(n => n.NodeName);
foreach (string node in machineNodes)
{
string nodeDirectory;
if (StandaloneUtility.DataRootNodeExists(machineName, node, fabricDataRoot, out nodeDirectory))
{
SFDeployerTrace.WriteError(StringResources.Error_BPADataRootNodeExists, node, machineName, nodeDirectory);
result = false;
}
}
}
}
if (!result)
{
if (Helpers.IsLocalIpAddress(machineName))
{
localMachineFailed = true;
}
else
{
machineHealthContainer.MarkMachineAsUnhealthy(machineName);
}
}
});
if (localMachineFailed)
{
return(false);
}
return(machineHealthContainer.EnoughHealthyMachines());
}
public async Task <string> ProcessSetUpgradeOrchestrationServiceStateAsync(string inputBlob, TimeSpan timeout, CancellationToken cancellationToken)
{
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Entering ProcessSetUpgradeOrchestrationServiceStateAsync.");
try
{
// data validation
StandAloneCluster cluster = null;
if (!string.IsNullOrWhiteSpace(inputBlob))
{
JsonSerializerSettings settings = StandaloneUtility.GetStandAloneClusterDeserializerSettings();
cluster = JsonConvert.DeserializeObject <StandAloneCluster>(
inputBlob,
settings);
}
await this.storeManager.SetStorageObjectAsync(Constants.ClusterReliableDictionaryKey, inputBlob, this.cancellationToken).ConfigureAwait(false);
FabricUpgradeOrchestrationServiceState result = UpgradeOrchestrationMessageProcessor.ConstructServiceStateFromCluster(cluster);
return(JsonConvert.SerializeObject(
result,
new JsonSerializerSettings
{
ReferenceLoopHandling = ReferenceLoopHandling.Serialize,
NullValueHandling = NullValueHandling.Ignore,
}));
}
catch (Exception e)
{
UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "ProcessSetUpgradeOrchestrationServiceStateAsync exception: {0}", e);
throw UpgradeOrchestrationMessageProcessor.ConvertToComException(e);
}
}
public async Task <string> ProcessGetClusterConfigurationAsync(string apiVersion, TimeSpan timeout, CancellationToken cancellationToken)
{
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Entering ProcessGetClusterConfigurationAsync.");
try
{
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Retrieve current cluster resource from StoreManager.");
StandAloneCluster cluster = await this.storeManager.GetClusterResourceAsync(
Constants.ClusterReliableDictionaryKey, this.cancellationToken).ConfigureAwait(false);
if (cluster == null || cluster.Current == null)
{
// read from fabric data root
string fabricDataRoot = FabricEnvironment.GetDataRoot();
string jsonConfigPath = Path.Combine(fabricDataRoot, System.Fabric.FabricDeployer.Constants.FileNames.BaselineJsonClusterConfig); // TODO: Needs to come from image store
return(File.ReadAllText(jsonConfigPath));
}
else
{
var nodesFromFM = new Dictionary <string, NodeDescription>();
NodeList nodes = await StandaloneUtility.GetNodesFromFMAsync(this.fabricClient, this.cancellationToken).ConfigureAwait(false);
for (int i = 0; i < nodes.Count; ++i)
{
if (nodes[i].NodeStatus != System.Fabric.Query.NodeStatus.Invalid && !UpgradeOrchestrationMessageProcessor.IsGhostNode(nodes[i]))
{
NodeDescription nodeDesc = new NodeDescription()
{
FaultDomain = nodes[i].FaultDomain.ToString(),
UpgradeDomain = nodes[i].UpgradeDomain,
IPAddress = nodes[i].IpAddressOrFQDN,
NodeTypeRef = nodes[i].NodeType,
NodeName = nodes[i].NodeName
};
nodesFromFM.Add(nodes[i].NodeName, nodeDesc);
}
}
cluster.Topology.Nodes = nodesFromFM;
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Construct StandAloneJsonModel from current cluster resource.");
var jsonModel = StandAloneInstallerJsonModelBase.ConstructByApiVersion(cluster.Current.CSMConfig, cluster.Topology, cluster.Current.CSMConfig.Version.Version, apiVersion);
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Serializing current json model.");
var serializerSettings = new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
PreserveReferencesHandling = PreserveReferencesHandling.None
};
serializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter());
return(JsonConvert.SerializeObject(jsonModel, serializerSettings));
}
}
catch (Exception e)
{
UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "ProcessStartUpgradeAsync exception: {0}", e);
throw UpgradeOrchestrationMessageProcessor.ConvertToComException(e);
}
}
internal static void UpdateStandAloneClusterWRPSettings(string clusterSettingsFilePath, string version, StandAloneCluster existingCluster)
{
string fabricSettingFilePath = StandaloneUtility.FindFabricResourceFile("Configurations.csv");
FabricSettingsMetadata fabricSettingsMetadata = FabricSettingsMetadata.Create(fabricSettingFilePath);
AdminConfigVersion adminConfigVersion = new AdminConfigVersion("Baseline", version);
StandAloneClusterManifestSettings standAloneClusterManifestSettings = JsonConvert.DeserializeObject <StandAloneClusterManifestSettings>(
File.ReadAllText(clusterSettingsFilePath));
existingCluster.TargetWrpConfig = new StandaloneAdminConfig(fabricSettingsMetadata, standAloneClusterManifestSettings, adminConfigVersion, false);
}
// This function check whether dotnet exe exists in %PATH% on IOT device.
// It will read %PATH% from registry and expand %systemroot% if found
// Currently it doesn't support expanding other variables.
internal static bool IsDotNetExeInPath(MachineHealthContainer machineHealthContainer)
{
bool result = true;
Parallel.ForEach(
machineHealthContainer.GetHealthyMachineNames(),
(string machineName) =>
{
// Currently we only need to check for IOTCore environment
if (StandaloneUtility.IsIOTCore(machineName))
{
string remotePath = string.Empty;
string remoteSystemRoot = string.Empty;
// Read %Path% variable
using (RegistryKey regKey = StandaloneUtility.GetHklm(machineName))
{
RegistryKey envKey = regKey.OpenSubKey(DMConstants.EnvironmentRegKeyPath);
if (envKey != null)
{
remotePath = (string)envKey.GetValue(DMConstants.Path, string.Empty, RegistryValueOptions.DoNotExpandEnvironmentNames);
}
RegistryKey versionKey = regKey.OpenSubKey(DMConstants.IOTCurrentVersionRegKeyPath);
if (versionKey != null)
{
remoteSystemRoot = (string)versionKey.GetValue(DMConstants.SystemRoot, string.Empty);
}
}
bool found = false;
string[] paths = remotePath.Split(';');
foreach (string path in paths)
{
// Replace %SystemRoot%
string absolutePath = Regex.Replace(path, "%SYSTEMROOT%", remoteSystemRoot, RegexOptions.IgnoreCase);
absolutePath = Path.Combine(absolutePath, "dotnet.exe");
string dotNetExePath = Helpers.GetRemotePathIfNotLocalMachine(absolutePath, machineName);
if (File.Exists(dotNetExePath))
{
found = true;
break;
}
}
if (!found)
{
result = false;
}
}
});
return(result);
}
public void DeploymentManagerTest_GetDevMachinesIsLocalhost()
{
string manifestPath = Path.Combine(BaseDir, XmlManifestFilename);
Assert.IsTrue(File.Exists(manifestPath), string.Format("Manifest file {0} did not exist.", manifestPath));
List <string> machines = StandaloneUtility.GetMachineNamesFromClusterManifest(manifestPath);
Assert.IsNotNull(machines);
Assert.AreEqual(machines.Count, 1);
Assert.IsTrue("localhost".CompareTo(machines[0]) == 0,
"Expected to find the machine localhost, but found {0} instead", machines[0]);
}
internal async Task PerformAddNodeOperationGMSAAsync(List <NodeDescription> addedNodes, FabricClient fabricClient, List <NodeTypeDescription> allNodeTypes)
{
string destinationPackagePath = await this.DownloadProvisionedPackage().ConfigureAwait(false);
var nodesFromFM = await StandaloneUtility.GetNodesFromFMAsync(fabricClient, this.cancellationToken).ConfigureAwait(false);
string clusterManifestPath = await this.GenerateClusterManifestWithAddedNodes(addedNodes, nodesFromFM, allNodeTypes);
List <string> machines = new List <string>();
foreach (var addednode in addedNodes)
{
if (!nodesFromFM.Any(nodeFromFM => nodeFromFM.NodeName == addednode.NodeName))
{
machines.Add(addednode.IPAddress);
}
}
bool successfulRun = true;
List <Exception> exceptions = new List <Exception>();
try
{
await this.AddNodesAsync(clusterManifestPath, destinationPackagePath, machines, addedNodes);
}
catch (Exception ex)
{
successfulRun = false;
exceptions.Add(ex);
UpgradeOrchestrationTrace.TraceSource.WriteError("Adding nodes failed with exception {0}.", ex.ToString());
}
if (!successfulRun)
{
// Best-Effort Rollback
try
{
await DeploymentManagerInternal.RemoveNodeConfigurationAsync(machines, false, FabricPackageType.XCopyPackage, true);
}
catch (Exception ex)
{
UpgradeOrchestrationTrace.TraceSource.WriteError("Rollback failed with {0}.", ex.ToString());
exceptions.Add(ex);
}
AggregateException ae = new AggregateException(exceptions);
throw ae;
}
this.CleanUp(clusterManifestPath, destinationPackagePath);
UpgradeOrchestrationTrace.TraceSource.WriteInfo("Nodes were succesfully added to the GMSA secured cluster");
}
private async Task <ReliabilityLevel> GetActualReliabilityLevelForCluster(CancellationToken cancellationToken)
{
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Entering GetCurrentReliabilityLevelForCluster");
System.Fabric.Query.NodeList nodes = await StandaloneUtility.GetNodesFromFMAsync(this.fabricClient, cancellationToken).ConfigureAwait(false);
int seedNodeCount = nodes.Count(p => p.IsSeedNode);
ReliabilityLevel actualReliabilityLevelInCluster = ReliabilityLevelExtension.GetReliabilityLevel(seedNodeCount);
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "SeedNodeCount from cluster {0}, actualReliabilityLevelInCluster {1}", seedNodeCount, actualReliabilityLevelInCluster);
return(actualReliabilityLevelInCluster);
}
internal static async Task <string> GetGoalStateFileJsonAsync(Uri goalStateUri)
{
string goalStateJson = null;
try
{
goalStateJson = await StandaloneUtility.GetContentsFromUriAsyncWithRetry(goalStateUri, TimeSpan.FromMinutes(Constants.FabricOperationTimeoutInMinutes), CancellationToken.None);
}
catch (FabricValidationException ex)
{
SFDeployerTrace.WriteError(StringResources.Error_SFGoalStateFileNotDownloaded, goalStateUri, ex.ToString());
}
return(goalStateJson);
}
internal static bool ValidateRemoveNode(string machineName)
{
SFDeployerTrace.WriteInfo(StringResources.Info_BPAStart);
//// TODO: (maburlik) Validate FabricClient connection
var summary = new AnalysisSummary();
summary.LocalAdminPrivilege = CheckLocalAdminPrivilege();
IEnumerable <string> machineContainer = new List <string>()
{
machineName
};
MachineHealthContainer machineHealthContainer = new MachineHealthContainer(machineContainer);
summary.RequiredPortsOpen = StandaloneUtility.CheckRequiredPorts(machineHealthContainer);
// Below depends on machines being reachable
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(false);
}
summary.RpcCheckPassed = CheckRPCAccess(machineHealthContainer);
// Below depend on remote registry access
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(false);
}
bool hasFabricInstalled = StandaloneUtility.IsFabricInstalled(machineName);
if (!hasFabricInstalled)
{
SFDeployerTrace.WriteError(StringResources.Error_BPARemoveNodeNeedsFabric, machineName);
}
LogResult(summary, hasFabricInstalled);
return(summary.Passed && hasFabricInstalled);
}
internal static bool ValidateClusterRemoval(
string configPath,
bool usingClusterManifest = false)
{
SFDeployerTrace.WriteInfo(StringResources.Info_BPAStart);
var summary = new AnalysisSummary();
// Check user has local admin privilege
summary.LocalAdminPrivilege = CheckLocalAdminPrivilege();
StandAloneInstallerJsonModelBase standAloneModel = null;
if (!usingClusterManifest)
{
standAloneModel = StandAloneInstallerJsonModelBase.GetJsonConfigFromFile(configPath);
summary.IsJsonValid = IsJsonConfigModelValid(standAloneModel, null, validateDownNodes: false, throwIfError: false);
}
// Below depends on JSON being valid
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(false);
}
// Get machine names from JSON config
IEnumerable <string> machineNames = usingClusterManifest
? StandaloneUtility.GetMachineNamesFromClusterManifest(configPath)
: standAloneModel.GetClusterTopology().Machines;
MachineHealthContainer machineHealthContainer = new MachineHealthContainer(machineNames);
// Log validations but don't fail
StandaloneUtility.CheckRequiredPorts(machineHealthContainer);
CheckRPCAccess(machineHealthContainer);
// At least one machine should be removable
bool anyMachinesRemovable = CheckAnyMachinesRemovable(machineNames);
LogResult(summary, anyMachinesRemovable);
return(summary.Passed);
}
public void DeploymentManagerTest_ConvertJsonToXml()
{
Console.WriteLine("Current directory: {0}", Environment.CurrentDirectory);
string clusterConfigPath = Path.Combine(BaseDir, JsonConfigUnsecureJan2017Filename);
Directory.SetCurrentDirectory(BaseDir);
Console.WriteLine("New current directory changed to: {0}", Environment.CurrentDirectory);
Assert.IsTrue(File.Exists(clusterConfigPath), string.Format("Config file {0} did not exist.", clusterConfigPath));
string clusterManifestPath = Path.Combine(Path.GetTempPath(), "cmtest.xml");
var clusterResource = DeploymentManagerInternal.GetClusterResource(clusterConfigPath, System.Guid.NewGuid().ToString());
var cm = clusterResource.Current.ExternalState.ClusterManifest;
Assert.IsNotNull(cm, "Cluster manifest was null.");
XMLHelper.WriteXmlExclusive <ClusterManifestType>(clusterManifestPath, cm);
Assert.IsTrue(File.Exists(clusterManifestPath), "Cluster manifest was not written.");
List <string> machines = StandaloneUtility.GetMachineNamesFromClusterManifest(clusterManifestPath);
Assert.IsNotNull(machines);
Assert.IsTrue(machines.Count > 0, "Cluster manifest was not written correctly. Machines missing.");
}
private static bool CheckAnyMachinesRemovable(IEnumerable <string> machineNames)
{
bool result = false;
Parallel.ForEach(
machineNames,
(string machineName) =>
{
if (StandaloneUtility.IsFabricInstalled(machineName))
{
result = true;
}
});
if (!result)
{
SFDeployerTrace.WriteError(StringResources.Error_BPANoMachinesHaveFabricInstalled);
}
return(result);
}
internal static StandAloneCluster DeserializeClusterResource(string str)
{
StandAloneCluster result;
JsonSerializerSettings settings = StandaloneUtility.GetStandAloneClusterDeserializerSettings();
try
{
result = JsonConvert.DeserializeObject <StandAloneCluster>(str, settings);
isVersionChangedFrom5To6 = false;
}
catch (JsonSerializationException ex)
{
UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "GetClusterResourceAsync: Deserialization error. Error is {0}. Will begin best-effort attempt.", ex.ToString());
var normalizedString = NormalizeClusterResource(str, Assembly.GetExecutingAssembly().GetName().Version.ToString());
result = JsonConvert.DeserializeObject <StandAloneCluster>(normalizedString, settings);
isVersionChangedFrom5To6 = true;
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "GetClusterResourceAsync: Deserialization succeeds after best-effort normalization.");
}
result.PersistedClusterString = str;
return(result);
}
private static bool CheckDrivesAvailableSpace(MachineHealthContainer machineHealthContainer, string fabricDataRoot, string fabricLogRoot)
{
List <string> drives = new List <string>();
string candidateMachine = machineHealthContainer.GetHealthyMachineNames().ElementAt(0);
string fabricPackageDestinationPath = Utility.GetDefaultPackageDestination(candidateMachine);
string fabricRootDrive = Path.GetPathRoot(fabricPackageDestinationPath);
drives.Add(fabricRootDrive);
if (fabricDataRoot != null)
{
drives.Add(Path.GetPathRoot(fabricDataRoot));
}
if (fabricLogRoot != null)
{
drives.Add(Path.GetPathRoot(fabricLogRoot));
}
foreach (string drive in drives.Distinct())
{
Parallel.ForEach(
machineHealthContainer.GetHealthyMachineNames(),
(string machineName) =>
{
string remoteRootDrive = Helpers.GetRemotePathIfNotLocalMachine(drive, machineName);
if (!StandaloneUtility.EnoughAvailableSpaceOnDrive(remoteRootDrive))
{
SFDeployerTrace.WriteError(StringResources.Error_BPANotEnoughSpaceOnDrive, drive, machineName);
machineHealthContainer.MarkMachineAsUnhealthy(machineName);
}
});
}
return(machineHealthContainer.EnoughHealthyMachines());
}
private static StandAloneCluster InitClusterResource(FabricNativeConfigStore configStore)
{
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Enter InitClusterResource");
string fabricDataRoot = FabricEnvironment.GetDataRoot();
string jsonConfigPath = Path.Combine(fabricDataRoot, Microsoft.ServiceFabric.DeploymentManager.Constants.BaselineJsonMetadataFileName);
ReleaseAssert.AssertIf(!File.Exists(jsonConfigPath), "Baseline upgrade JsonClusterConfigMetadata not found at {0}.", jsonConfigPath);
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Deserializing Json config.");
string jsonConfig = File.ReadAllText(jsonConfigPath);
if (string.IsNullOrEmpty(jsonConfig))
{
throw new FabricValidationException(StringResources.Error_SFJsonConfigInvalid, FabricErrorCode.OperationCanceled);
}
JsonSerializerSettings settings = StandaloneUtility.GetStandAloneClusterDeserializerSettings();
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Exit InitClusterResource");
return(JsonConvert.DeserializeObject <StandAloneCluster>(
jsonConfig,
settings));
}
internal static bool ValidateAddNode(string machineName, string nodeName, string cabPath, string fabricDataRoot, string fabricLogRoot)
{
SFDeployerTrace.WriteInfo(StringResources.Info_BPAStart);
//// TODO: (maburlik) Validate FabricClient connection
var summary = new AnalysisSummary();
// Check user has local admin privilege
summary.LocalAdminPrivilege = CheckLocalAdminPrivilege();
summary.IsCabValid = CheckIsCabFile(cabPath);
// Below depends on JSON being valid
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(false);
}
IEnumerable <string> machineContainer = new List <string>()
{
machineName
};
MachineHealthContainer machineHealthContainer = new MachineHealthContainer(machineContainer);
summary.RequiredPortsOpen = StandaloneUtility.CheckRequiredPorts(machineHealthContainer);
// Below depends on machines being reachable
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(false);
}
summary.RpcCheckPassed = CheckRPCAccess(machineHealthContainer);
// Below depend on remote registry access
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(false);
}
summary.NoDomainController = CheckNoMachineIsDomainController(machineHealthContainer);
// Below depends on machine not being domain controller
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(false);
}
summary.NoConflictingInstallations = !StandaloneUtility.IsMsiInstalled(machineHealthContainer);
summary.FabricInstallable = StandaloneUtility.CheckForCleanInstall(machineName, nodeName, fabricDataRoot);
summary.DataDrivesAvailable = CheckDataSystemDrives(machineHealthContainer, fabricDataRoot, fabricLogRoot);
// Below depend on data system drives
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(false);
}
summary.DrivesEnoughAvailableSpace = CheckDrivesAvailableSpace(machineHealthContainer, fabricDataRoot, fabricLogRoot);
// Below depend on root drives having enough space available
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(false);
}
LogResult(summary);
return(summary.Passed);
}
internal void ConvertDevJsonToXml_Utility(int expectedReplicaSize, string inputFile, string outputFile, string sectionToChangeWin7, bool isSecure = false, bool testOverride = false)
{
Console.WriteLine("Current directory: {0}", Environment.CurrentDirectory);
string clusterConfigPath = Path.Combine(DevJsonModelTest.BaseDir, inputFile);
Directory.SetCurrentDirectory(DevJsonModelTest.BaseDir);
Console.WriteLine("New current directory changed to: {0}", Environment.CurrentDirectory);
string clusterManifestPath = Path.Combine(Path.GetTempPath(), outputFile);
var clusterResource = DeploymentManagerInternal.GetClusterResource(clusterConfigPath, System.Guid.NewGuid().ToString());
var cm = clusterResource.Current.ExternalState.ClusterManifest;
Assert.IsNotNull(cm, "Cluster manifest was null.");
File.Delete(clusterManifestPath);
XMLHelper.WriteXmlExclusive <ClusterManifestType>(clusterManifestPath, cm);
List <string> machines = StandaloneUtility.GetMachineNamesFromClusterManifest(clusterManifestPath);
Assert.IsNotNull(machines);
Assert.AreEqual(expectedReplicaSize, ((WindowsInfrastructureType)cm.Infrastructure.Item).NodeList.ToList().Where(node => node.IsSeedNode).Count());
if (isSecure)
{
Assert.AreEqual(InputEndpointTypeProtocol.https, cm.NodeTypes[0].Endpoints.HttpGatewayEndpoint.Protocol);
Assert.IsNotNull(cm.NodeTypes[0].Certificates.ClientCertificate, "Client certificate was null.");
}
foreach (SettingsOverridesTypeSection setting in cm.FabricSettings)
{
Assert.IsTrue(IsSectionAllowed(cm.NodeTypes.Count(), setting.Name));
if (setting.Name == StringConstants.SectionName.Setup)
{
foreach (SettingsOverridesTypeSectionParameter param in setting.Parameter)
{
Assert.AreNotEqual(StringConstants.ParameterName.IsDevCluster, param.Name);
if (param.Name == StringConstants.ParameterName.FabricDataRoot)
{
Assert.AreEqual(Environment.ExpandEnvironmentVariables("%systemdrive%\\SfDevCluster\\Data"), param.Value);
}
}
}
if (setting.Name == StringConstants.SectionName.ClusterManager)
{
SettingsOverridesTypeSectionParameter TargetReplicaSetSizeParam = setting.Parameter.FirstOrDefault(x => x.Name == StringConstants.ParameterName.TargetReplicaSetSize);
Assert.IsNotNull(TargetReplicaSetSizeParam, "Cannot find TargetReplicaSetSize for Cluster Manager");
Assert.AreEqual(expectedReplicaSize.ToString(), TargetReplicaSetSizeParam.Value);
}
if (DevJsonModel.IsWin7() && setting.Name == sectionToChangeWin7)
{
SettingsOverridesTypeSectionParameter isEnabledParam = setting.Parameter.FirstOrDefault(x => x.Name == StringConstants.ParameterName.IsEnabled);
Assert.IsNotNull(isEnabledParam, "Cannot find isEnabled for " + sectionToChangeWin7);
Assert.AreEqual("false", isEnabledParam.Value);
}
if (setting.Name == StringConstants.SectionName.Diagnostics && testOverride)
{
SettingsOverridesTypeSectionParameter isEnabledTraceSessionParam = setting.Parameter.FirstOrDefault(x => x.Name == StringConstants.ParameterName.EnableCircularTraceSession);
Assert.IsNotNull(isEnabledTraceSessionParam, "Cannot find isEnabled for EnableCircularTraceSession");
Assert.AreEqual("false", isEnabledTraceSessionParam.Value);
}
if (setting.Name == StringConstants.SectionName.FailoverManager && expectedReplicaSize == 1)
{
SettingsOverridesTypeSectionParameter IsSingletonReplicaMoveAllowedDuringUpgradeParam = setting.Parameter.FirstOrDefault(x => x.Name == StringConstants.ParameterName.IsSingletonReplicaMoveAllowedDuringUpgrade);
Assert.IsNotNull(IsSingletonReplicaMoveAllowedDuringUpgradeParam, "Cannot find isEnabled for EnableCircularTraceSession");
Assert.AreEqual("false", IsSingletonReplicaMoveAllowedDuringUpgradeParam.Value);
}
}
//DevCluster should have one distinct IPaddress, i.e. localhost
Assert.AreEqual(1, machines.Count);
}
private static bool CheckRPCAccess(MachineHealthContainer machineHealthContainer)
{
var retryTimeout = new System.Fabric.Common.TimeoutHelper(DMConstants.BpaRpcRetryTimeout);
SFDeployerTrace.WriteNoise(StringResources.Info_SFRpcInfo);
Parallel.ForEach <string>(
machineHealthContainer.GetHealthyMachineNames(),
(string machine) =>
{
bool result = true;
bool willRetry;
do
{
willRetry = false;
try
{
Utility.GetTempPath(machine);
}
catch (Exception ex)
{
string message;
if (ex is System.IO.IOException)
{
switch (ex.HResult)
{
// If new failures are discovered: https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx
case 53: // ERROR_BAD_NETPATH
message = string.Format(StringResources.Error_SFRpcIoNetpath, machine, ex.HResult);
willRetry = true;
break;
case 1723: // RPC_S_SERVER_TOO_BUSY
message = string.Format(StringResources.Error_SFRpcIoTooBusy, machine, ex.HResult);
willRetry = true;
break;
case 1727: // RPC_S_CALL_FAILED_DNE
message = string.Format(StringResources.Error_SFRpcIoFailedDne, machine, ex.HResult);
break;
default:
message = string.Format(StringResources.Error_SFRpcIoGeneric, machine, ex.HResult);
break;
}
}
else if (ex is System.Security.SecurityException)
{
switch (ex.HResult)
{
case -2146233078: // COR_E_SECURITY
message = string.Format(StringResources.Error_SFRpcSecAccess, machine, ex.HResult);
break;
default:
message = string.Format(StringResources.Error_SFRpcSecGeneric, machine, ex.HResult);
break;
}
}
else if (ex is NullReferenceException)
{
switch (ex.HResult)
{
case -2146232828: // COR_E_TARGETINVOCATION
message = string.Format(StringResources.Error_SFRpcNullRegAccess, machine, ex.HResult);
break;
default:
message = string.Format(StringResources.Error_SFRpcNullGeneric, machine, ex.HResult);
break;
}
}
else
{
// This is to catch coding errors.
message = string.Format(StringResources.Error_SFRpcGeneric, machine, ex.HResult);
}
willRetry &= !System.Fabric.Common.TimeoutHelper.HasExpired(retryTimeout);
if (willRetry)
{
SFDeployerTrace.WriteWarning(message);
StandaloneUtility.OpenRemoteRegistryNamedPipe(machine, retryTimeout.GetRemainingTime());
Thread.Sleep(TimeSpan.FromSeconds(5));
}
else
{
SFDeployerTrace.WriteError(message);
result = false;
}
}
}while (willRetry);
if (!result)
{
machineHealthContainer.MarkMachineAsUnhealthy(machine);
}
});
return(machineHealthContainer.EnoughHealthyMachines());
}
internal static AnalysisSummary AnalyzeClusterSetup(
string configPath,
string oldConfigPath,
string cabPath,
bool usingClusterManifest = false,
FabricPackageType fabricPackageType = FabricPackageType.XCopyPackage,
bool isForcedRun = false,
int maxPercentFailedNodes = 0)
{
SFDeployerTrace.WriteInfo(StringResources.Info_BPAStart);
var summary = new AnalysisSummary();
// Check user has local admin privilege
summary.LocalAdminPrivilege = CheckLocalAdminPrivilege();
// Validate JSON config
StandAloneInstallerJsonModelBase standAloneModel = null;
StandAloneInstallerJsonModelBase oldStandAloneModel = null;
if (!usingClusterManifest)
{
standAloneModel = StandAloneInstallerJsonModelBase.GetJsonConfigFromFile(configPath);
if (!string.IsNullOrEmpty(oldConfigPath))
{
oldStandAloneModel = StandAloneInstallerJsonModelBase.GetJsonConfigFromFile(oldConfigPath);
}
summary.IsJsonValid = IsJsonConfigModelValid(standAloneModel, oldStandAloneModel, validateDownNodes: true, throwIfError: false) && ValidateCodeVersionExists(standAloneModel);
}
// Deliberately not checking empty. Some invocations aren't intended to test cabPath.
if (cabPath != null)
{
summary.IsCabValid = CheckIsCabFile(cabPath);
}
// Below depends on JSON being valid
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(summary);
}
// Get machine names from JSON config
IEnumerable <string> healthyMachineNames = usingClusterManifest
? StandaloneUtility.GetMachineNamesFromClusterManifest(configPath)
: standAloneModel.GetClusterTopology().Machines;
MachineHealthContainer machineHealthContainer = new MachineHealthContainer(healthyMachineNames, maxPercentFailedNodes);
// Validate machine FQDNs, Check SMB ports opened
summary.RequiredPortsOpen = StandaloneUtility.CheckRequiredPorts(machineHealthContainer);
// Below depends on machines being reachable
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(summary);
}
// Validate Remote Registry Service is not disabled on all machines
summary.RemoteRegistryAvailable = CheckRemoteRegistryEnabled(machineHealthContainer);
// Below depends on remote registry service
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(summary);
}
summary.FirewallAvailable = CheckFirewallEnabled(machineHealthContainer);
// Below depends on firewall service
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(summary);
}
// Run RPC check to validate end-to-end registry access to all machines
summary.RpcCheckPassed = CheckRPCAccess(machineHealthContainer);
// Below depend on remote registry access
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(summary);
}
summary.NoDomainController = CheckNoMachineIsDomainController(machineHealthContainer);
// Below depends on machines not being domain controller
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(summary);
}
if (fabricPackageType == FabricPackageType.XCopyPackage)
{
// Validate that Fabric runtime MSI is not installed since this will be conflicting with Standalone
summary.NoConflictingInstallations = !StandaloneUtility.IsMsiInstalled(machineHealthContainer);
}
string fabricDataRoot = null;
string fabricLogRoot = null;
if (!usingClusterManifest)
{
if (string.IsNullOrEmpty(oldConfigPath))
{
summary.FabricInstallable = CheckForCleanInstall(standAloneModel, machineHealthContainer, isForcedRun); // Fabric is not installed on target machines
}
var importantSettings = standAloneModel.GetFabricSystemSettings();
fabricDataRoot = importantSettings.ContainsKey(DMConstants.FabricDataRootString) ?
importantSettings[DMConstants.FabricDataRootString] :
null;
fabricLogRoot = importantSettings.ContainsKey(DMConstants.FabricLogRootString) ?
importantSettings[DMConstants.FabricLogRootString] :
null;
summary.DataDrivesAvailable = CheckDataSystemDrives(machineHealthContainer, fabricDataRoot, fabricLogRoot); // System drives for path-based settings, exist on target machines
// Below depend on data system drives
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(summary);
}
}
summary.DrivesEnoughAvailableSpace = CheckDrivesAvailableSpace(machineHealthContainer, fabricDataRoot, fabricLogRoot);
// Below depend on root drives having enough space available
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(summary);
}
summary.IsAllOrNoneIOTDevice = CheckAllIOTDevices(machineHealthContainer);
// Below depend on all or none machines being IOT Devices
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(summary);
}
// Check dotnet.exe exists in %Path%
// Currently we only need to check for IOTCore environment.
summary.DotNetExeInPath = IsDotNetExeInPath(machineHealthContainer);
// Below depend on IOT Devices having dotnet.exe in path
if (!summary.Passed)
{
SFDeployerTrace.WriteError(StringResources.Error_BPABailing);
return(summary);
}
summary.MachineHealthContainer = machineHealthContainer;
LogResult(summary);
return(summary);
}
public async Task PersistClusterResourceAsync(string key, StandAloneCluster cluster, CancellationToken cancellationToken)
{
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Enter PersistClusterResourceAsync.");
var clusterStateTable = await this.stateManager.GetOrAddAsync <IReliableDictionary <string, string> >(Constants.ClusterReliableDictionaryName);
try
{
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Serialize cluster resource into string format.");
string clusterString = JsonConvert.SerializeObject(
cluster,
StandaloneUtility.GetStandAloneClusterSerializerSettings());
using (var tx = this.stateManager.CreateTransaction())
{
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Enter transaction inside PersistClusterResourceAsync");
if (string.IsNullOrEmpty(cluster.PersistedClusterString))
{
await clusterStateTable.AddAsync(
tx,
key,
clusterString,
Constants.DefaultDictionaryTimeout,
cancellationToken).ConfigureAwait(false);
}
else
{
/* This is a workaround for scenarios where clusters were upgraded from 5.x->6.x. During deserealization, we change the version in UOSState from 5.0.0.0 to 6.0.0.0. This is needed for
* newtonsoft deserealization to work with the new assembly versions. Here we revert 6.0.0.0 back to 5.0.0.0 before persisting it so that users can downgrade succesfully from 6.x -> 5.x
* versions. Once 5.x is deprecated, we can remove this workaround and persist 6.0.0.0 in UOS state */
if (isVersionChangedFrom5To6)
{
clusterString = NormalizeClusterResource(clusterString, Version5);
}
bool success = await clusterStateTable.TryUpdateAsync(
tx,
key,
clusterString,
cluster.PersistedClusterString).ConfigureAwait(false);
if (!success)
{
UpgradeOrchestrationTrace.TraceSource.WriteWarning(
TraceType,
"PersistClusterResourceAsync: StandAloneCluster data is not updated. The reason is {0}",
string.Equals(clusterString, cluster.PersistedClusterString, StringComparison.OrdinalIgnoreCase) ? "the value is not changed." : "to be investigated.");
return;
}
}
await tx.CommitAsync().ConfigureAwait(false);
}
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Exit transaction inside PersistClusterResourceAsync");
}
catch (Exception e)
{
UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "PersistClusterResourceAsync: {0}", e.ToString());
throw;
}
}
public void DeploymentManagerTest_ProductionGoalState()
{
Console.WriteLine("test begins: {0}", DateTime.UtcNow);
string goalStateFilepath = Path.Combine(BaseDir, GoalStateFilename);
Assert.IsTrue(File.Exists(goalStateFilepath), string.Format("GoalStateFile {0} did not exist.", goalStateFilepath));
string goalStateJson = File.ReadAllText(goalStateFilepath);
GoalStateModel model = GoalStateModel.GetGoalStateModelFromJson(goalStateJson);
Assert.IsTrue(model != null, "Goal state JSON could not be deserialized.");
Assert.IsTrue(model.Packages != null, "Goal state model must have a packages element.");
if (model.Packages.Count() == 0)
{
return;
}
// There can only be one goal package
int countGoalpackages = model.Packages.Where(package => package.IsGoalPackage).Count();
Assert.IsTrue(countGoalpackages == 1, "Exactly one goal package may be marked as IsGoalPackage in the goal state file.");
// Validate each package link is reachable
string errorMessage = string.Empty;
foreach (PackageDetails package in model.Packages)
{
Uri packageUri;
if (!Uri.TryCreate(package.TargetPackageLocation, UriKind.Absolute, out packageUri))
{
errorMessage = string.Format("Cannot parse packageUri: {0}", package.TargetPackageLocation);
Assert.Fail(errorMessage);
}
if (!StandaloneUtility.IsUriReachable(packageUri))
{
errorMessage = string.Format("Package uri is not reachable: {0}", packageUri.AbsoluteUri);
}
}
if (!string.IsNullOrEmpty(errorMessage))
{
Assert.Fail(errorMessage);
}
// Validate that goal state downloaded package is the correct version
string packageDropDir = System.Fabric.Common.Helpers.GetNewTempPath();
PackageDetails goalPackage = model.Packages.First(package => package.IsGoalPackage);
Assert.IsTrue(goalPackage != null, "goalPackage null");
// Validate package downloadable
Console.WriteLine("download begins for {0}: {1}", goalPackage.Version, DateTime.UtcNow);
string packageLocalDownloadPath = Path.Combine(packageDropDir, string.Format(Microsoft.ServiceFabric.DeploymentManager.Constants.SFPackageDropNameFormat, goalPackage.Version));
bool packageDownloaded = StandaloneUtility.DownloadPackageAsync(goalPackage.Version, goalPackage.TargetPackageLocation, packageLocalDownloadPath).Result;
Assert.IsTrue(packageDownloaded, string.Format("Package {0} failed to download from {1}.", goalPackage.Version, goalPackage.TargetPackageLocation));
// Validate downloaded package matches version of version in goal state file
string cabVersion = CabFileOperations.GetCabVersion(packageLocalDownloadPath);
Assert.IsTrue(goalPackage.Version == cabVersion, string.Format("Goal state package version {0} doesn't match downloaded package version {1}.", goalPackage.Version, cabVersion));
if (Directory.Exists(packageDropDir))
{
try
{
FabricDirectory.Delete(packageDropDir, true, true);
}
catch { }
}
Console.WriteLine("test ends: {0}", DateTime.UtcNow);
}
private async Task PollGoalStateForCodePackagesAsync(CancellationToken cancellationToken)
{
var fabricClient = new FabricClient();
//// region: Initialization Parameters
FabricUpgradeProgress upgradeProgress;
try
{
upgradeProgress = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync <FabricUpgradeProgress>(
() =>
fabricClient.ClusterManager.GetFabricUpgradeProgressAsync(TimeSpan.FromMinutes(DMConstants.FabricOperationTimeoutInMinutes), cancellationToken),
TimeSpan.FromMinutes(DMConstants.FabricOperationTimeoutInMinutes),
cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
this.traceSource.WriteError(TraceType, "GetFabricUpgradeProgressAsync threw: {0}", ex.ToString());
this.timerInterval = TimeSpan.FromMinutes(DMConstants.FabricUpgradeStatePollIntervalInMinutes);
return;
}
if (!FabricClientWrapper.IsUpgradeCompleted(upgradeProgress.UpgradeState))
{
this.timerInterval = TimeSpan.FromMinutes(DMConstants.FabricUpgradeStatePollIntervalInMinutes);
this.traceSource.WriteInfo(TraceType, "Cannot retrieve cluster version; FabricUpgradeState is currently: {0}. Will retry in {1}.", upgradeProgress.UpgradeState.ToString(), this.timerInterval.ToString());
return;
}
string currentVersionStr = upgradeProgress.TargetCodeVersion;
Version currentVersion = Version.Parse(currentVersionStr);
string packageDropDir = System.Fabric.Common.Helpers.GetNewTempPath();
NativeConfigStore configStore = NativeConfigStore.FabricGetConfigStore();
string goalStateUriStr = configStore.ReadUnencryptedString(DMConstants.UpgradeOrchestrationServiceConfigSectionName, DMConstants.GoalStateFileUrlName);
if (string.IsNullOrEmpty(goalStateUriStr))
{
goalStateUriStr = DMConstants.DefaultGoalStateFileUrl;
}
//// endregion
this.traceSource.WriteInfo(TraceType, "PollCodePackagesFromGoalState currentVersion: {0}, packageDropDir: {1} goalStateUri: {2}", currentVersionStr, packageDropDir, goalStateUriStr);
try
{
Uri goalStateUri = null;
string goalStateJson = null;
if (!Uri.TryCreate(goalStateUriStr, UriKind.Absolute, out goalStateUri))
{
string errorMessage = string.Format("Cannot parse GoalStateUri: {0}", goalStateUriStr);
this.traceSource.WriteError(TraceType, errorMessage);
ReleaseAssert.Fail(errorMessage);
return;
}
if (!(await StandaloneUtility.IsUriReachableAsync(goalStateUri).ConfigureAwait(false)))
{
this.timerInterval = TimeSpan.FromMinutes(DMConstants.FabricGoalStateReachablePollIntervalInMinutes);
this.traceSource.WriteWarning(TraceType, "Cannot reach download uri for goal state file: {0}. Will retry in {1}.", goalStateUri.AbsoluteUri, this.timerInterval.ToString());
this.EmitGoalStateReachableHealth(fabricClient, false);
return;
}
else
{
this.EmitGoalStateReachableHealth(fabricClient, true);
}
goalStateJson = await GoalStateModel.GetGoalStateFileJsonAsync(goalStateUri).ConfigureAwait(false);
if (string.IsNullOrEmpty(goalStateJson))
{
this.traceSource.WriteWarning(TraceType, "Loaded goal state JSON is empty.");
return;
}
GoalStateModel model = GoalStateModel.GetGoalStateModelFromJson(goalStateJson);
if (model == null || model.Packages == null)
{
this.traceSource.WriteWarning(TraceType, "Goal state JSON could not be deserialized:\n{0}", goalStateJson);
return;
}
string availableGoalStateVersions = string.Format("Available versions in goal state file: {0}", model.ToString());
this.traceSource.WriteInfo(TraceType, availableGoalStateVersions);
IEnumerable <PackageDetails> candidatePackages = model.Packages.Where(
package => currentVersion < Version.Parse(package.Version));
if (candidatePackages.Count() == 0)
{
this.traceSource.WriteInfo(TraceType, "No upgrades available.");
}
string versionCurrentStr = currentVersionStr;
Version versionCurrent = currentVersion;
PackageDetails targetPackage = null;
for (IEnumerable <PackageDetails> availableCandidatePackages = candidatePackages.Where(
package => (package.MinVersion == null || Version.Parse(package.MinVersion) <= versionCurrent));
availableCandidatePackages.Count() > 0;
versionCurrentStr = targetPackage.Version,
versionCurrent = Version.Parse(versionCurrentStr),
availableCandidatePackages = candidatePackages.Where(
package => (versionCurrent < Version.Parse(package.Version) &&
(package.MinVersion == null || Version.Parse(package.MinVersion) <= versionCurrent))))
{
if (!IsVersionUpgradeable(versionCurrentStr, model))
{
this.traceSource.WriteInfo(TraceType, "Version {0} is not upgradeable.", versionCurrentStr);
break;
}
int numPackages = availableCandidatePackages.Count();
if (numPackages == 0)
{
this.traceSource.WriteInfo(TraceType, "No upgrade available.");
return;
}
else if (numPackages == 1)
{
targetPackage = availableCandidatePackages.First();
}
else
{
Version maxVersion = StandaloneGoalStateProvisioner.ZeroVersion;
foreach (PackageDetails package in availableCandidatePackages)
{
Version targetVersion;
if (!Version.TryParse(package.Version, out targetVersion))
{
this.traceSource.WriteWarning(TraceType, "Package {0} version could not be parsed. Trying another one.", package.Version);
continue;
}
if (targetVersion > maxVersion)
{
targetPackage = package;
maxVersion = targetVersion;
}
}
}
try
{
if (await this.IsPackageProvisionedAsync(targetPackage.Version, fabricClient, cancellationToken).ConfigureAwait(false))
{
this.traceSource.WriteInfo(TraceType, "Package {0} is already provisioned.", targetPackage.Version);
continue;
}
}
catch (Exception ex)
{
this.traceSource.WriteError(TraceType, "PackageIsProvisioned for {0} threw: {1}", targetPackage.Version, ex.ToString());
}
string packageLocalDownloadPath = Path.Combine(packageDropDir, string.Format(DMConstants.SFPackageDropNameFormat, targetPackage.Version));
if (await StandaloneUtility.DownloadPackageAsync(targetPackage.Version, targetPackage.TargetPackageLocation, packageLocalDownloadPath).ConfigureAwait(false))
{
try
{
this.traceSource.WriteInfo(TraceType, "Copying and provisioning version {0} from {1}.", targetPackage.Version, packageLocalDownloadPath);
await ProvisionPackageAsync(targetPackage.Version, packageLocalDownloadPath, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
this.traceSource.WriteError(TraceType, "ProvisionPackageAsync for {0}, package {1} threw: {2}", targetPackage.Version, packageLocalDownloadPath, ex.ToString());
}
}
}
// Determine if current version is latest/doesn't have expiry date. If package expiring, post cluster health warning.
PackageDetails currentPackage = model.Packages.Where(package => package.Version == currentVersionStr).FirstOrDefault();
if (currentPackage != null && currentPackage.SupportExpiryDate != null && !currentPackage.IsGoalPackage && !this.IsAutoupgradeInstallEnabled() && this.IsPackageNearExpiry(currentPackage))
{
this.traceSource.WriteWarning(TraceType, "Current version {0} expires {1}; emitting cluster warning.", currentVersionStr, currentPackage.SupportExpiryDate);
this.EmitClusterVersionSupportedHealth(fabricClient, false, currentVersionStr, currentPackage.SupportExpiryDate);
}
else
{
this.traceSource.WriteInfo(TraceType, "Current version {0} is supported. Cluster health OK.", currentVersionStr);
this.EmitClusterVersionSupportedHealth(fabricClient, true, null, null);
}
if (targetPackage != null &&
!string.IsNullOrEmpty(targetPackage.Version) &&
this.IsAutoupgradeInstallEnabled())
{
try
{
// fire off an upgrade.
this.StartCodeUpgrade(targetPackage.Version, fabricClient);
}
catch (Exception ex)
{
this.traceSource.WriteError(TraceType, "StartCodeUpgrade for version {0} threw: {1}", targetPackage.Version, ex.ToString());
}
}
}
finally
{
if (Directory.Exists(packageDropDir))
{
try
{
FabricDirectory.Delete(packageDropDir, true, true);
}
catch
{
}
}
}
}
public async Task ProcessStartClusterConfigurationUpgradeAsync(ConfigurationUpgradeDescription configUpgradeDesc, TimeSpan timeout, CancellationToken cancellationToken)
{
/* The cancellation token passed in this API call is not used (by design). This token corresponds to the client side call.
* The global this.cancellationToken is initialised in RunAsync() and is honored in every API call. */
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Entering ProcessStartUpgradeAsync.");
try
{
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Deserializing input json config string.");
StandAloneInstallerJsonModelBase targetJsonConfig = StandAloneInstallerJsonModelBase.GetJsonConfigFromString(configUpgradeDesc.ClusterConfiguration);
if (targetJsonConfig == null)
{
throw new ArgumentException("The input cluster configuration is not in a valid json format or supported apiVersion.");
}
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Retrieve current cluster resource from StoreManager.");
StandAloneCluster cluster = await this.storeManager.GetClusterResourceAsync(
Constants.ClusterReliableDictionaryKey, this.cancellationToken).ConfigureAwait(false);
bool isInDataLossState = await DatalossHelper.IsInDatalossStateAsync(this.cancellationToken).ConfigureAwait(false);
if (!isInDataLossState)
{
if (cluster == null || cluster.Current == null)
{
UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "Persisted cluster resource is not ready: {0}", cluster == null ? "null" : "current = null");
throw new FabricException("UpgradeOrchestrationService is not ready.");
}
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Setting target config and topology based on new input json config.");
if (cluster.Pending != null && !this.IsInterruptibleAsync(cluster.Pending).Result)
{
throw new FabricException(string.Format("Cluster configuration upgrade of type {0} is already in progress and cannot be interrupted.", cluster.Pending.GetType().Name));
}
StandaloneSettingsValidator validator = new StandaloneSettingsValidator(targetJsonConfig);
await UpgradeOrchestrationMessageProcessor.ValidateModel(targetJsonConfig, validator, cluster, true).ConfigureAwait(false);
var removedNodes = validator.GetRemovedNodes(cluster.Topology);
var addedNodes = validator.GetAddedNodes(cluster.Topology);
if (addedNodes.Any() && StandaloneUtility.CheckFabricRunningAsGMSA(cluster.Current.CSMConfig))
{
/* Need to resolve assembly so that FabricDeployer can load the right binaries from FabricCodePath since not all binaries required by FabricDeployer are present in UOS.Current folder.*/
AppDomain.CurrentDomain.AssemblyResolve += new ResolveEventHandler(this.LoadFromFabricCodePath);
try
{
await this.PerformAddNodeOperationGMSAAsync(addedNodes, this.fabricClient, validator.ClusterProperties.NodeTypes).ConfigureAwait(false);
}
catch (AggregateException ex)
{
UpgradeOrchestrationTrace.TraceSource.WriteError(TraceType, "Adding nodes for GMSA scenario failed with exception: {0}", ex);
throw;
}
finally
{
AppDomain.CurrentDomain.AssemblyResolve -= new ResolveEventHandler(this.LoadFromFabricCodePath);
}
}
if (addedNodes.Any())
{
cluster.TargetNodeConfig = GetTargetNodeConfigAddNode(validator.Topology, cluster.Current.NodeConfig.Version);
}
if (removedNodes.Any())
{
cluster.TargetNodeConfig = GetTargetNodeConfigRemoveNode(cluster.Topology, removedNodes, cluster.Current.NodeConfig.Version);
}
else
{
cluster.Topology = validator.Topology;
}
cluster.TargetCsmConfig = validator.ClusterProperties;
// Cluster is updated above so persist it.
await this.storeManager.PersistClusterResourceAsync(Constants.ClusterReliableDictionaryKey, cluster, cancellationToken).ConfigureAwait(false);
await this.UpdatePersistedCodeUpgradePackage(validator).ConfigureAwait(false);
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Invoking Orchestrator");
await this.orchestrator.StartUpgradeAsync(cluster, this.cancellationToken, configUpgradeDesc).ContinueWith(t =>
{
if (t.Exception != null)
{
UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "Orchestrator completed with status: {0} exception: {1}", t.Status, t.Exception);
}
else
{
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Orchestrator completed with status: {0}", t.Status);
}
});
}
else
{
StandaloneSettingsValidator validator = new StandaloneSettingsValidator(targetJsonConfig);
await UpgradeOrchestrationMessageProcessor.ValidateModel(targetJsonConfig, validator, cluster, false).ConfigureAwait(false);
cluster = FabricUpgradeOrchestrationService.ConstructClusterFromJson(targetJsonConfig, FabricNativeConfigStore.FabricGetConfigStore());
DatalossHelper.DryRunConfigUpgrade(cluster);
await this.storeManager.PersistClusterResourceAsync(Constants.ClusterReliableDictionaryKey, cluster, this.cancellationToken);
await DatalossHelper.UpdateHeathStateAsync(isHealthy : true).ConfigureAwait(false);
}
}
catch (Exception e)
{
UpgradeOrchestrationTrace.TraceSource.WriteWarning(TraceType, "ProcessStartUpgradeAsync exception: {0}", e);
throw UpgradeOrchestrationMessageProcessor.ConvertToComException(e);
}
UpgradeOrchestrationTrace.TraceSource.WriteInfo(TraceType, "Exiting ProcessStartUpgradeAsync.");
}
internal static bool CheckAllIOTDevices(MachineHealthContainer machineHealthContainer)
{
int numOfIOTCores = StandaloneUtility.IOTDevicesCount(machineHealthContainer.GetHealthyMachineNames());
return(numOfIOTCores == 0 || numOfIOTCores == machineHealthContainer.GetHealthyMachineNames().Count());
}
