private static ClusterTopology GetEnvironmentTopology()
{
string machineInfoFile = Path.Combine(APRuntime.DataDirectory, StringConstants.MachineInfoFileName);
List <InfrastructureNodeType> serviceFabricNodes = new List <InfrastructureNodeType>();
using (CsvReader reader = new CsvReader(machineInfoFile))
{
while (reader.Read())
{
var serviceFabricNode = new InfrastructureNodeType()
{
NodeName = reader["MachineName"],
// Note that NodeTypeRef and RoleOrTierName are provided due to XML schema of infrastructure manifest.
NodeTypeRef = reader["MachineFunction"],
RoleOrTierName = reader["MachineFunction"],
IPAddressOrFQDN = reader["MachineName"],
UpgradeDomain = reader["ScaleUnit"],
FaultDomain = string.Format(CultureInfo.InvariantCulture, "fd:/{0}", reader["PodName"])
};
serviceFabricNodes.Add(serviceFabricNode);
}
};
return(new ClusterTopology
{
ServiceFabricNodes = serviceFabricNodes,
CurrentNodeName = string.Empty
});
}
internal static ClusterTopology GetClusterTopologyFromServiceRuntime(LocalNodeConfiguration localWindowsFabricNode)
{
List <InfrastructureNodeType> windowsFabricNodes = new List <InfrastructureNodeType>();
int applicationPortCount = localWindowsFabricNode.ApplicationEndPort - localWindowsFabricNode.ApplicationStartPort + 1;
Dictionary <string, WindowsFabricEndpointSet> roleNameToEndpointSetMap = GetRoleNameToEndpointSetMap(applicationPortCount);
WindowsFabricEndpointSet currentRoleEndpointSet = roleNameToEndpointSetMap[RoleEnvironment.CurrentRoleInstance.Role.Name];
foreach (KeyValuePair <string, Role> role in RoleEnvironment.Roles)
{
foreach (RoleInstance roleInstance in role.Value.Instances.Where(instance => IsWindowsFabricRoleInstance(instance)))
{
var roleEndpointSet = roleNameToEndpointSetMap[roleInstance.Role.Name];
InfrastructureNodeType node = new InfrastructureNodeType()
{
NodeName = GetNodeName(roleInstance),
UpgradeDomain = roleInstance.UpdateDomain.ToString(CultureInfo.InvariantCulture),
FaultDomain = string.Format(CultureInfo.InvariantCulture, WindowsFabricAzureWrapperServiceCommon.FaultDomainTemplate, roleInstance.FaultDomain),
IPAddressOrFQDN = GetRoleInstanceHostIPAddress(roleInstance).ToString(),
RoleOrTierName = roleInstance.Role.Name,
// The following two entries are unknown at this time and will be populated by FabricDeployer based on cluster manifest
NodeTypeRef = string.Empty,
IsSeedNode = false,
Endpoints = new FabricEndpointsType()
{
ClientConnectionEndpoint = roleEndpointSet.ClientConnectionEndpoint ?? currentRoleEndpointSet.ClientConnectionEndpoint,
HttpGatewayEndpoint = roleEndpointSet.HttpGatewayEndpoint ?? currentRoleEndpointSet.HttpGatewayEndpoint,
HttpApplicationGatewayEndpoint = roleEndpointSet.HttpApplicationGatewayEndpoint ?? currentRoleEndpointSet.HttpApplicationGatewayEndpoint,
ClusterConnectionEndpoint = roleEndpointSet.ClusterConnectionEndpoint,
LeaseDriverEndpoint = roleEndpointSet.LeaseDriverEndpoint,
ApplicationEndpoints = roleEndpointSet.ApplicationEndpoints,
}
};
windowsFabricNodes.Add(node);
}
}
// node list is always ordered by node name
windowsFabricNodes.Sort(CompareNodeByName);
return(new ClusterTopology
{
WindowsFabricNodes = windowsFabricNodes,
CurrentNodeName = GetNodeName(RoleEnvironment.CurrentRoleInstance),
CurrentNodeIPAddressOrFQDN = GetRoleInstanceHostIPAddress(RoleEnvironment.CurrentRoleInstance).ToString(),
});
}
private static bool NodeEquals(InfrastructureNodeType nodeA, InfrastructureNodeType nodeB)
{
if (string.Compare(nodeA.NodeName, nodeB.NodeName, StringComparison.OrdinalIgnoreCase) != 0 ||
string.Compare(nodeA.UpgradeDomain, nodeB.UpgradeDomain, StringComparison.OrdinalIgnoreCase) != 0 ||
string.Compare(nodeA.FaultDomain, nodeB.FaultDomain, StringComparison.OrdinalIgnoreCase) != 0 ||
string.Compare(nodeA.IPAddressOrFQDN, nodeB.IPAddressOrFQDN, StringComparison.OrdinalIgnoreCase) != 0 ||
string.Compare(nodeA.RoleOrTierName, nodeB.RoleOrTierName, StringComparison.OrdinalIgnoreCase) != 0)
{
return(false);
}
return(FabricEndpointsEquals(nodeA.Endpoints, nodeB.Endpoints));
}
public static bool IsNodeForThisMachine(InfrastructureNodeType node)
{
IPAddress ipAddress;
if (IPAddress.TryParse(node.IPAddressOrFQDN, out ipAddress))
{
return(FabricValidatorUtility.CurrentMachineAddresses.Contains(ipAddress) ||
IsAddressLoopback(node.IPAddressOrFQDN));
}
else
{
return(FabricValidatorUtility.CurrentMachineNames.Contains(node.IPAddressOrFQDN));
}
}
private static bool NodeEquals(InfrastructureNodeType nodeA, InfrastructureNodeType nodeB)
{
// TODO: On Autopilot, endpoint set is not part of service model so would not be modeled as part of topology.
if (string.Compare(nodeA.NodeName, nodeB.NodeName, StringComparison.Ordinal) != 0 ||
string.Compare(nodeA.UpgradeDomain, nodeB.UpgradeDomain, StringComparison.OrdinalIgnoreCase) != 0 ||
string.Compare(nodeA.FaultDomain, nodeB.FaultDomain, StringComparison.OrdinalIgnoreCase) != 0 ||
string.Compare(nodeA.IPAddressOrFQDN, nodeB.IPAddressOrFQDN, StringComparison.OrdinalIgnoreCase) != 0 ||
string.Compare(nodeA.RoleOrTierName, nodeB.RoleOrTierName, StringComparison.OrdinalIgnoreCase) != 0)
{
return(false);
}
return(true);
}
public static List <InfrastructureNodeType> Create(string ServiceVMComputerName)
{
ServiceVMComputerName = ServiceVMComputerName.Trim();
if (ServiceVMComputerName.Length == 0)
{
return(null);
}
var nodeList = new List <InfrastructureNodeType>();
string[] tiers = ServiceVMComputerName.Split("]".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
System.Console.WriteLine("tiers are: {0}", tiers);
foreach (string tier in tiers)
{
if (tier.Trim().Length == 0)
{
continue;
}
string[] tierInfo = tier.Split('[');
string tierName = tierInfo[0];
string[] nodes = tierInfo[1].Split(',');
foreach (string node in nodes)
{
string[] nodeinfo = node.Split(':');
string nodeName = nodeinfo[0];
string upgradDomain = nodeinfo[1];
InfrastructureNodeType n = new InfrastructureNodeType
{
FaultDomain = GetFaultDomain(nodeName),
UpgradeDomain = upgradDomain,
IPAddressOrFQDN = nodeName,
NodeName = nodeName,
RoleOrTierName = tierName,
IsSeedNode = true,
NodeTypeRef = string.Empty
};
nodeList.Add(n);
}
}
return(nodeList);
}
internal static bool IsNodeForThisMachine(InfrastructureNodeType node)
{
return(IsAddressForThisMachine(node.IPAddressOrFQDN));
}
public static bool IsNodeAddressLoopback(InfrastructureNodeType node)
{
return(IsAddressLoopback(node.IPAddressOrFQDN));
}
private static int CompareNodeByName(InfrastructureNodeType nodeA, InfrastructureNodeType nodeB)
{
return(string.Compare(nodeA.NodeName, nodeB.NodeName, StringComparison.OrdinalIgnoreCase));
}
private void RemoveLogicalDirectoriesIfExist(string fabricDataRoot, string machineName)
{
if (machineName == null)
{
machineName = "";
}
ClusterManifestType clusterManifest = null;
var clusterManifestLocation = Helpers.GetCurrentClusterManifestPath(fabricDataRoot);
if (clusterManifestLocation == null)
{
clusterManifestLocation = Directory.EnumerateFiles(
fabricDataRoot,
"clusterManifest.xml",
SearchOption.AllDirectories).FirstOrDefault();
}
if (clusterManifestLocation != null)
{
clusterManifest = XMLHelper.ReadClusterManifest(clusterManifestLocation);
}
else
{
DeployerTrace.WriteWarning(string.Format(StringResources.Warning_clusterManifestFileMissingInDataRoot, fabricDataRoot, machineName));
}
if (clusterManifest != null)
{
InfrastructureInformationType infrastructure = null;
FabricNodeType[] nodeList;
if ((clusterManifest.Infrastructure.Item is ClusterManifestTypeInfrastructureWindowsServer &&
((ClusterManifestTypeInfrastructureWindowsServer)clusterManifest.Infrastructure.Item).IsScaleMin) ||
(clusterManifest.Infrastructure.Item is ClusterManifestTypeInfrastructureLinux &&
((ClusterManifestTypeInfrastructureLinux)clusterManifest.Infrastructure.Item).IsScaleMin))
{
DeployerTrace.WriteInfo(string.Format("Checking to remove logicalDirectories for one box scale min deployment. Machine name: {0}.", machineName));
// Get all the nodes on this IP's LogicalDirectories and remove the all directoies.
nodeList = clusterManifest.Infrastructure.Item is ClusterManifestTypeInfrastructureWindowsServer ?
((ClusterManifestTypeInfrastructureWindowsServer)clusterManifest.Infrastructure.Item).NodeList :
((ClusterManifestTypeInfrastructureLinux)clusterManifest.Infrastructure.Item).NodeList;
RemoveLogicalDirectoies(clusterManifest, nodeList, machineName);
}
else if (clusterManifest.Infrastructure.Item is ClusterManifestTypeInfrastructurePaaS) // PaaS doesn't support scale min.
{
var infrastructureManifestLocation = Helpers.GetInfrastructureManifestPath(fabricDataRoot);
if (infrastructureManifestLocation == null)
{
DeployerTrace.WriteWarning(string.Format(StringResources.Warning_infrastructureManifestFileMissingInDataRoot, fabricDataRoot, machineName));
return;
}
infrastructure = XMLHelper.ReadInfrastructureManifest(infrastructureManifestLocation);
// Get all the nodes on this IP's LogicalDirectories and remove the all directories.
var len = infrastructure.NodeList.Length;
nodeList = new FabricNodeType[len];
for (var i = 0; i < len; i++)
{
FabricNodeType fabricNode = new FabricNodeType()
{
NodeName = infrastructure.NodeList[i].NodeName,
FaultDomain = infrastructure.NodeList[i].FaultDomain,
IPAddressOrFQDN = infrastructure.NodeList[i].IPAddressOrFQDN,
IsSeedNode = infrastructure.NodeList[i].IsSeedNode,
NodeTypeRef = infrastructure.NodeList[i].NodeTypeRef,
UpgradeDomain = infrastructure.NodeList[i].UpgradeDomain
};
nodeList[i] = fabricNode;
}
RemoveLogicalDirectoies(clusterManifest, nodeList, machineName);
}
else if (clusterManifest.Infrastructure.Item is ClusterManifestTypeInfrastructureWindowsServer ||
clusterManifest.Infrastructure.Item is ClusterManifestTypeInfrastructureLinux)
{
nodeList = clusterManifest.Infrastructure.Item is ClusterManifestTypeInfrastructureWindowsServer ?
((ClusterManifestTypeInfrastructureWindowsServer)clusterManifest.Infrastructure.Item).NodeList :
((ClusterManifestTypeInfrastructureLinux)clusterManifest.Infrastructure.Item).NodeList;
if (!string.IsNullOrEmpty(machineName)) //For cab deployment
{
DeployerTrace.WriteInfo(string.Format("Checking to remove logicalDirectories for WindowsServer/Linux deployment with cab. Machine name: {0}.", machineName));
RemoveLogicalDirectoies(clusterManifest, nodeList, machineName);
}
else
{
DeployerTrace.WriteInfo(string.Format("Checking to remove logicalDirectories for WindowsServer/Linux deployment with MSI. Machine name: {0}.", machineName));
foreach (var node in nodeList)
{
InfrastructureNodeType infraNode = new InfrastructureNodeType()
{
NodeName = node.NodeName,
NodeTypeRef = node.NodeTypeRef,
IPAddressOrFQDN = node.IPAddressOrFQDN,
IsSeedNode = node.IsSeedNode,
FaultDomain = node.FaultDomain,
UpgradeDomain = node.UpgradeDomain
};
bool isNodeForThisMachine = NetworkApiHelper.IsNodeForThisMachine(infraNode);
if (isNodeForThisMachine)
{
FabricNodeType[] fabricNodes = new FabricNodeType[1];
FabricNodeType fabricNode = new FabricNodeType()
{
NodeName = node.NodeName,
FaultDomain = node.FaultDomain,
IPAddressOrFQDN = node.IPAddressOrFQDN,
IsSeedNode = node.IsSeedNode,
NodeTypeRef = node.NodeTypeRef,
UpgradeDomain = node.UpgradeDomain
};
fabricNodes[0] = fabricNode;
RemoveLogicalDirectoies(clusterManifest, fabricNodes, machineName);
}
}
}
}
}
}
private void InitializeInfrastructureSettings(
int nNodeTypes,
bool isScaleMin,
bool isLogicalDirectories,
int[] nNodes,
int[] nSeedNodes,
int[] nFaultDomains,
int[] nUpgradDomains,
InfrastructureType infrastructureType)
{
this.nodeTypes = new ClusterManifestTypeNodeType[nNodeTypes];
int totalNodes = 0;
for (int i = 0; i < nNodeTypes; i++)
{
totalNodes += nNodes[i];
}
this.nodes = new FabricNodeType[totalNodes];
this.infraNodes = new InfrastructureNodeType[totalNodes];
int nodeIndex = 0;
for (int i = 0; i < nNodeTypes; i++)
{
int clientConnectionPort = 19000 + i;
int leaseDriverPort = 19000 + nNodeTypes + i;
int clusterConnectionPort = 19000 + 2 * nNodeTypes + i;
int httpGatewayPort = 19000 + 3 * nNodeTypes + i;
int serviceConnectionPort = 19000 + 4 * nNodeTypes + i;
int startAppPort = 30000 + 1000 * i + 1;
int endAppPort = 30000 + 1000 * (i + 1);
string host = isScaleMin ? "localhost" : System.Net.Dns.GetHostName();
string roleOrTierName = GetRoleOrTiername(i);
string faultDomain = string.Format(CultureInfo.InvariantCulture, "fd:/Rack{0}", i % nFaultDomains[i]);
string upgradeDomain = string.Format(CultureInfo.InvariantCulture, "MYUD{0}", i % nUpgradDomains[i]);
FabricEndpointsType endpoints = new FabricEndpointsType()
{
ApplicationEndpoints = new FabricEndpointsTypeApplicationEndpoints()
{
StartPort = startAppPort, EndPort = endAppPort
},
LeaseDriverEndpoint = new InternalEndpointType()
{
Port = leaseDriverPort.ToString(CultureInfo.InvariantCulture)
},
ClusterConnectionEndpoint = new InternalEndpointType()
{
Port = clusterConnectionPort.ToString(CultureInfo.InvariantCulture)
},
ClientConnectionEndpoint = new InputEndpointType()
{
Port = clientConnectionPort.ToString(CultureInfo.InvariantCulture)
},
HttpGatewayEndpoint = new InputEndpointType()
{
Port = httpGatewayPort.ToString(CultureInfo.InvariantCulture)
},
ServiceConnectionEndpoint = new InternalEndpointType()
{
Port = serviceConnectionPort.ToString(CultureInfo.InvariantCulture)
}
};
for (int j = 0; j < nNodes[i]; j++)
{
bool isSeedNode = j < nSeedNodes[i];
string nodeName = string.Format(CultureInfo.InvariantCulture, "{0}.Node.{1}", roleOrTierName, j);
FabricNodeType node = new FabricNodeType()
{
NodeName = nodeName, FaultDomain = faultDomain, IPAddressOrFQDN = host, IsSeedNode = isSeedNode, NodeTypeRef = roleOrTierName, UpgradeDomain = upgradeDomain
};
InfrastructureNodeType infraNode = new InfrastructureNodeType()
{
NodeName = nodeName, FaultDomain = faultDomain, IPAddressOrFQDN = host, IsSeedNode = isSeedNode, NodeTypeRef = roleOrTierName, UpgradeDomain = upgradeDomain, RoleOrTierName = roleOrTierName, Certificates = null, Endpoints = endpoints
};
this.nodes[nodeIndex] = node;
this.infraNodes[nodeIndex] = infraNode;
nodeIndex++;
}
nodeTypes[i] = new ClusterManifestTypeNodeType();
nodeTypes[i].Certificates = null;
if (infrastructureType == InfrastructureType.WindowsServer || infrastructureType == InfrastructureType.VMM || infrastructureType == InfrastructureType.PaaS)
{
nodeTypes[i].Endpoints = endpoints;
}
nodeTypes[i].Name = roleOrTierName;
}
if (isLogicalDirectories)
{
nodeTypes[0].LogicalDirectories = new LogicalDirectoryType[5];
nodeTypes[0].LogicalDirectories[0] = new LogicalDirectoryType
{
LogicalDirectoryName = "0",
MappedTo = TestUtility.LogicalDirectoriesApplicationCheckPointFilesDir,
Context = LogicalDirectoryTypeContext.node
};
nodeTypes[0].LogicalDirectories[1] = new LogicalDirectoryType
{
LogicalDirectoryName = "1",
MappedTo = TestUtility.LogicalDirectoriesLogDir,
Context = LogicalDirectoryTypeContext.node
};
nodeTypes[0].LogicalDirectories[2] = new LogicalDirectoryType
{
LogicalDirectoryName = "2",
MappedTo = TestUtility.LogicalDirectoriesBackupDir,
Context = LogicalDirectoryTypeContext.application
};
nodeTypes[0].LogicalDirectories[3] = new LogicalDirectoryType
{
LogicalDirectoryName = "3",
MappedTo = TestUtility.LogicalDirectoriesUserDefined1Dir
};
nodeTypes[0].LogicalDirectories[4] = new LogicalDirectoryType
{
LogicalDirectoryName = "4",
MappedTo = TestUtility.LogicalDirectoriesUserDefined2Dir,
Context = LogicalDirectoryTypeContext.node
};
nodeTypes[1].LogicalDirectories = new LogicalDirectoryType[5];
nodeTypes[1].LogicalDirectories[0] = new LogicalDirectoryType
{
LogicalDirectoryName = "0",
MappedTo = TestUtility.LogicalDirectoriesApplicationCheckPointFilesDir2,
Context = LogicalDirectoryTypeContext.application
};
nodeTypes[1].LogicalDirectories[1] = new LogicalDirectoryType
{
LogicalDirectoryName = "1",
MappedTo = TestUtility.LogicalDirectoriesLogDir2,
Context = LogicalDirectoryTypeContext.application
};
nodeTypes[1].LogicalDirectories[2] = new LogicalDirectoryType
{
LogicalDirectoryName = "2",
MappedTo = TestUtility.LogicalDirectoriesBackupDir2,
Context = LogicalDirectoryTypeContext.node
};
nodeTypes[1].LogicalDirectories[3] = new LogicalDirectoryType
{
LogicalDirectoryName = "3",
MappedTo = TestUtility.LogicalDirectoriesUserDefined1Dir2,
Context = LogicalDirectoryTypeContext.application
};
nodeTypes[1].LogicalDirectories[4] = new LogicalDirectoryType
{
LogicalDirectoryName = "4",
MappedTo = TestUtility.LogicalDirectoriesUserDefined2Dir2,
Context = LogicalDirectoryTypeContext.application
};
}
}
private void InitializeFromServiceRuntimeTopologyProvider(ClusterManifestTypeInfrastructureWindowsAzure azureInfrastructure, InfrastructureNodeType[] inputInfrastructureNodes, Dictionary <string, ClusterManifestTypeNodeType> nameToNodeTypeMap)
{
// For service runtime topology provider, source of cluster topology is infrastructure manifest provided by azure wrapper service.
// Seed nodes are determined as the first SeedNodeCount nodes of the seed node role.
ReleaseAssert.AssertIfNull(inputInfrastructureNodes, "inputInfrastructureNodes");
Dictionary <string, int> seedInfo = new Dictionary <string, int>();
Dictionary <string, string> roleToTypeMap = new Dictionary <string, string>();
foreach (var role in azureInfrastructure.Roles)
{
try
{
seedInfo.Add(role.RoleName, role.SeedNodeCount);
roleToTypeMap.Add(role.RoleName, role.NodeTypeRef);
}
catch (ArgumentException)
{
throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, "Role name {0} is duplicate", role.RoleName));
}
}
foreach (var nodeInfo in inputInfrastructureNodes)
{
string nodeIndex = nodeInfo.NodeName.Split(".".ToCharArray()).Last();
int index = int.Parse(nodeIndex);
ClusterManifestTypeNodeType nodeType = null;
if (roleToTypeMap.ContainsKey(nodeInfo.RoleOrTierName))
{
var nodeTypeName = roleToTypeMap[nodeInfo.RoleOrTierName];
if (nameToNodeTypeMap.ContainsKey(nodeTypeName))
{
nodeType = nameToNodeTypeMap[nodeTypeName];
}
else
{
throw new ArgumentException(
string.Format(
CultureInfo.InvariantCulture,
"Node type {0} specified for role {1} in node {2} not found",
nodeTypeName,
nodeInfo.RoleOrTierName,
nodeInfo.NodeName));
}
}
else
{
throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, "Role name {0} specified in node {1} not found", nodeInfo.NodeName, nodeInfo.RoleOrTierName));
}
var node = new InfrastructureNodeType()
{
FaultDomain = nodeInfo.FaultDomain,
IPAddressOrFQDN = nodeInfo.IPAddressOrFQDN,
IsSeedNode = index < seedInfo[nodeInfo.RoleOrTierName],
NodeName = nodeInfo.NodeName,
NodeTypeRef = roleToTypeMap[nodeInfo.RoleOrTierName],
UpgradeDomain = nodeInfo.UpgradeDomain,
RoleOrTierName = nodeInfo.RoleOrTierName,
Endpoints = nodeInfo.Endpoints,
Certificates = nodeType.Certificates
};
this.nodes.Add(node);
if (node.IsSeedNode)
{
// Add vote mappings
string connectionAddress = string.Format(CultureInfo.InvariantCulture, "{0}:{1}", node.IPAddressOrFQDN, node.Endpoints.ClusterConnectionEndpoint.Port);
this.voteMappings.Add(node.NodeName, connectionAddress);
FabricValidator.TraceSource.WriteNoise(FabricValidatorUtility.TraceTag, "Mapping {0}-{1} added", node.NodeName, connectionAddress);
// Add votes.
string key = node.NodeName;
string value = string.Format(CultureInfo.InvariantCulture, "{0},{1}", "WindowsAzure", node.NodeName);
this.votes.Add(new SettingsTypeSectionParameter()
{
Name = key, Value = value, MustOverride = false
});
FabricValidator.TraceSource.WriteNoise(FabricValidatorUtility.TraceTag, "Vote {0}-{1} added", key, value);
string clientConnectionAddress = string.Format(CultureInfo.InvariantCulture, "{0}:{1}", node.IPAddressOrFQDN, node.Endpoints.ClientConnectionEndpoint.Port);
this.seedNodeClientConnectionAddresses.Add(new SettingsTypeSectionParameter()
{
Name = key, Value = clientConnectionAddress, MustOverride = false
});
FabricValidator.TraceSource.WriteNoise(FabricValidatorUtility.TraceTag, "Seed Nodes {0}-{1} added", key, clientConnectionAddress);
}
}
this.isScaleMin = FabricValidatorUtility.IsAddressLoopback(nodes[0].IPAddressOrFQDN);
}