public void AddressCount()
{
Assert.Equal(Math.Pow(2, 32), NetworkCidr.Parse("10.0.0.1/0").AddressCount);
Assert.Equal(Math.Pow(2, 24), NetworkCidr.Parse("10.0.0.1/8").AddressCount);
Assert.Equal(Math.Pow(2, 16), NetworkCidr.Parse("10.0.0.1/16").AddressCount);
Assert.Equal(Math.Pow(2, 8), NetworkCidr.Parse("10.0.0.1/24").AddressCount);
}
public void ContainsSubnet()
{
Assert.True(NetworkCidr.Parse("10.0.0.0/24").Contains(NetworkCidr.Parse("10.0.0.0/24")));
Assert.True(NetworkCidr.Parse("10.0.0.0/24").Contains(NetworkCidr.Parse("10.0.0.0/25")));
Assert.False(NetworkCidr.Parse("10.0.0.0/24").Contains(NetworkCidr.Parse("10.0.0.0/23")));
Assert.False(NetworkCidr.Parse("10.0.0.0/24").Contains(NetworkCidr.Parse("10.0.2.0/24")));
}
/// <summary>
/// <para>
/// Lists the virtual IPv4 addresses.
/// </para>
/// <note>
/// Only IPv4 addresses are returned. IPv6 and any other address types will be ignored.
/// </note>
/// </summary>
/// <returns>A list of <see cref="VirtualIPAddress"/>.</returns>
public List <VirtualIPAddress> ListIPAddresses()
{
CheckDisposed();
try
{
var addresses = new List <VirtualIPAddress>();
var rawAddresses = powershell.ExecuteJson($"{NetTcpIpNamespace}Get-NetIPAddress");
var switchRegex = new Regex(@"^.*\((?<switch>.+)\)$");
foreach (dynamic rawAddress in rawAddresses)
{
// We're only listing IPv4 addresses.
var address = (string)rawAddress.IPv4Address;
if (string.IsNullOrEmpty(address))
{
continue;
}
// Extract the interface/switch name from the [InterfaceAlias] field,
// which will look something like:
//
// vEthernet (neonkube)
//
// We'll extract the name within the parens if present, otherwise we'll
// take the entire property value as the name.
var interfaceAlias = (string)rawAddress.InterfaceAlias;
var match = switchRegex.Match(interfaceAlias);
var interfaceName = string.Empty;
if (match.Success)
{
interfaceName = match.Groups["switch"].Value;
}
else
{
interfaceName = interfaceAlias;
}
var virtualIPAddress
= new VirtualIPAddress()
{
Address = address,
Subnet = NetworkCidr.Parse($"{address}/{rawAddress.PrefixLength}"),
InterfaceName = interfaceName
};
addresses.Add(virtualIPAddress);
}
return(addresses);
}
catch (Exception e)
{
throw new HyperVException(e.Message, e);
}
}
public void MaskedAddress()
{
var subnet = NetworkCidr.Parse("10.0.1.5/24");
Assert.Equal(IPAddress.Parse("255.255.255.0"), subnet.Mask);
Assert.Equal(IPAddress.Parse("10.0.1.0"), subnet.Address);
}
public void ParseErrors()
{
Assert.Throws <ArgumentNullException>(() => NetworkCidr.Parse(null));
Assert.Throws <ArgumentNullException>(() => NetworkCidr.Parse(string.Empty));
Assert.Throws <ArgumentException>(() => NetworkCidr.Parse("10.0.0.1"));
Assert.Throws <ArgumentException>(() => NetworkCidr.Parse("/6"));
Assert.Throws <ArgumentException>(() => NetworkCidr.Parse("10.0.0.1/-1"));
Assert.Throws <ArgumentException>(() => NetworkCidr.Parse("10.0.0.1/33"));
Assert.Throws <ArgumentException>(() => NetworkCidr.Parse("10.A.0.1/8"));
}
/// <summary>
/// Validates the address rule.
/// </summary>
/// <param name="clusterDefinition">The cluster definition.</param>
/// <param name="context">Indicates where the address rule is specified, like: <b>ingress-rule-address</b> or <b>egress-address</b></param>
/// <exception cref="ClusterDefinitionException">Thrown for an invalid rule.</exception>
public void Validate(ClusterDefinition clusterDefinition, string context)
{
if (!IsAny)
{
if (!NetHelper.TryParseIPv4Address(AddressOrSubnet, out var v1) && !NetworkCidr.TryParse(AddressOrSubnet, out var v2))
{
throw new ClusterDefinitionException($"Invalid address or subnet [{AddressOrSubnet}] specified for a [{context}].");
}
}
}
public void ContainsIP()
{
Assert.True(NetworkCidr.Parse("10.0.0.0/24").Contains(IPAddress.Parse("10.0.0.0")));
Assert.True(NetworkCidr.Parse("10.0.0.0/24").Contains(IPAddress.Parse("10.0.0.1")));
Assert.True(NetworkCidr.Parse("10.0.0.0/24").Contains(IPAddress.Parse("10.0.0.255")));
Assert.True(NetworkCidr.Parse("10.0.1.0/24").Contains(IPAddress.Parse("10.0.1.1")));
Assert.False(NetworkCidr.Parse("10.0.0.0/24").Contains(IPAddress.Parse("10.0.1.0")));
Assert.False(NetworkCidr.Parse("10.0.0.0/24").Contains(IPAddress.Parse("10.0.1.1")));
Assert.False(NetworkCidr.Parse("10.0.0.0/24").Contains(IPAddress.Parse("10.0.1.255")));
}
public void Overlaps()
{
var subnet = NetworkCidr.Parse("10.0.1.0/24");
Assert.True(subnet.Overlaps(NetworkCidr.Parse("10.0.1.0/24")));
Assert.True(subnet.Overlaps(NetworkCidr.Parse("10.0.1.0/16")));
Assert.True(subnet.Overlaps(NetworkCidr.Parse("10.0.0.0/23")));
Assert.True(subnet.Overlaps(NetworkCidr.Parse("10.0.1.16/28")));
Assert.False(subnet.Overlaps(NetworkCidr.Parse("10.0.0.0/24")));
Assert.False(subnet.Overlaps(NetworkCidr.Parse("10.0.2.0/24")));
}
public void TryParseErrors()
{
NetworkCidr cidr;
Assert.False(NetworkCidr.TryParse(null, out cidr));
Assert.False(NetworkCidr.TryParse(string.Empty, out cidr));
Assert.False(NetworkCidr.TryParse("10.0.0.1", out cidr));
Assert.False(NetworkCidr.TryParse("/6", out cidr));
Assert.False(NetworkCidr.TryParse("10.0.0.1/-1", out cidr));
Assert.False(NetworkCidr.TryParse("10.0.0.1/33", out cidr));
Assert.False(NetworkCidr.TryParse("10.A.0.1/8", out cidr));
}
//---------------------------------------------------------------------
// VirtualIPAddress
/// <summary>
/// Converts a <see cref="GrpcVirtualIPAddress"/> into a <see cref="VirtualIPAddress"/>.
/// </summary>
/// <param name="grpcVirtualIPAddress">The input.</param>
/// <returns>The output.</returns>
public static VirtualIPAddress?ToLocal(this GrpcVirtualIPAddress grpcVirtualIPAddress)
{
if (grpcVirtualIPAddress == null)
{
return(null);
}
return(new VirtualIPAddress()
{
Address = grpcVirtualIPAddress.Address,
Subnet = NetworkCidr.Parse(grpcVirtualIPAddress.Subnet),
InterfaceName = grpcVirtualIPAddress.InterfaceName
});
}
/// <summary>
/// Adds an internal Hyper-V switch configured for the specified subnet and gateway as well
/// as an optional NAT enabling external connectivity.
/// </summary>
/// <param name="switchName">The new switch name.</param>
/// <param name="subnet">Specifies the internal subnet.</param>
/// <param name="addNat">Optionally configure a NAT to support external routing.</param>
public void NewInternalSwitch(string switchName, NetworkCidr subnet, bool addNat = false)
{
if (isAdmin)
{
hypervClient.NewInternalSwitch(switchName: switchName, subnet: subnet, addNat: addNat);
}
else
{
var request = new GrpcNewInternalSwitchRequest(switchName: switchName, subnet: subnet, addNat: addNat);
var reply = desktopService.NewInternalSwitchAsync(request).Result;
reply.Error.EnsureSuccess();
}
}
/// <summary>
/// Validates the options and also ensures that all <c>null</c> properties are
/// initialized to their default values.
/// </summary>
/// <param name="clusterDefinition">The cluster definition.</param>
/// <exception cref="ClusterDefinitionException">Thrown if the definition is not valid.</exception>
public void Validate(ClusterDefinition clusterDefinition)
{
Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));
var kubernetesOptionsPrefix = $"{nameof(ClusterDefinition.Kubernetes)}";
Version = Version ?? defaultVersion;
Version = Version.ToLowerInvariant();
if (Version != defaultVersion)
{
if (!System.Version.TryParse(Version, out var kubernetesVersion))
{
throw new ClusterDefinitionException($"[{kubernetesOptionsPrefix}.{nameof(Version)}={Version}] is not a valid Kubernetes version.");
}
if (kubernetesVersion < System.Version.Parse(minVersion))
{
throw new ClusterDefinitionException($"[{kubernetesOptionsPrefix}.{nameof(Version)}={Version}] is less than the supported version [{minVersion}].");
}
}
if (DashboardVersion != defaultDashboardVersion)
{
if (!System.Version.TryParse(DashboardVersion, out var vDashboard))
{
throw new ClusterDefinitionException($"[{kubernetesOptionsPrefix}.{nameof(DashboardVersion)}={DashboardVersion}] is not a valid version number.");
}
}
if (HelmVersion != "default" && !System.Version.TryParse(HelmVersion, out var vHelm))
{
throw new ClusterDefinitionException($"[{kubernetesOptionsPrefix}.{nameof(HelmVersion)}={HelmVersion}] is invalid].");
}
if (!AllowPodsOnControlPlane.HasValue)
{
AllowPodsOnControlPlane = clusterDefinition.Workers.Count() == 0;
}
var podSubnetCidr = NetworkCidr.Parse(clusterDefinition.Network.PodSubnet);
if ((clusterDefinition.Nodes.Count() * MaxPodsPerNode * 2.3) > podSubnetCidr.UsableAddressCount)
{
var maxPods = podSubnetCidr.UsableAddressCount / 2.3;
var clusterPods = clusterDefinition.Nodes.Count() * MaxPodsPerNode;
var maxPodsPerNode = maxPods / clusterDefinition.Nodes.Count();
var maxNodes = maxPods / MaxPodsPerNode;
throw new ClusterDefinitionException(@$ "[{kubernetesOptionsPrefix}.{nameof(MaxPodsPerNode)}={MaxPodsPerNode}] is not valid.
public void FirstLastAndNext()
{
var subnet = NetworkCidr.Parse("127.0.0.0/24");
Assert.Equal("127.0.0.0", subnet.FirstAddress.ToString());
Assert.Equal("127.0.0.1", subnet.FirstUsableAddress.ToString());
Assert.Equal("127.0.0.255", subnet.LastAddress.ToString());
Assert.Equal("127.0.1.0", subnet.NextAddress.ToString());
subnet = NetworkCidr.Parse("10.0.1.0/16");
Assert.Equal("10.0.0.0", subnet.FirstAddress.ToString());
Assert.Equal("10.0.255.255", subnet.LastAddress.ToString());
Assert.Equal("10.1.0.0", subnet.NextAddress.ToString());
}
/// <inheritdoc/>
public async Task <GrpcBaseReply> NewInternalSwitchAsync(GrpcNewInternalSwitchRequest request, CallContext context = default)
{
await SyncContext.Clear;
try
{
hyperv.NewInternalSwitch(
switchName: request.SwitchName,
subnet: NetworkCidr.Parse(request.Subnet),
addNat: request.AddNat);
return(new GrpcBaseReply());
}
catch (Exception e)
{
return(new GrpcBaseReply(e));
}
}
/// <summary>
/// Adds an internal Hyper-V switch configured for the specified subnet and gateway as well
/// as an optional NAT enabling external connectivity.
/// </summary>
/// <param name="switchName">The new switch name.</param>
/// <param name="subnet">Specifies the internal subnet.</param>
/// <param name="addNat">Optionally configure a NAT to support external routing.</param>
public void NewInternalSwitch(string switchName, NetworkCidr subnet, bool addNat = false)
{
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(switchName), nameof(switchName));
Covenant.Requires <ArgumentNullException>(subnet != null, nameof(subnet));
CheckDisposed();
var gatewayAddress = subnet.FirstUsableAddress;
powershell.Execute($"{HyperVNamespace}New-VMSwitch -Name '{switchName}' -SwitchType Internal");
powershell.Execute($"{NetTcpIpNamespace}New-NetIPAddress -IPAddress {subnet.FirstUsableAddress} -PrefixLength {subnet.PrefixLength} -InterfaceAlias 'vEthernet ({switchName})'");
if (addNat)
{
if (GetNatByName(switchName) == null)
{
powershell.Execute($"{NetNatNamespace}New-NetNAT -Name '{switchName}' -InternalIPInterfaceAddressPrefix {subnet}");
}
}
}
public void Init()
{
var cidr = new NetworkCidr(IPAddress.Parse("10.1.2.3"), 8);
Assert.Equal(IPAddress.Parse("10.0.0.0"), cidr.Address);
Assert.Equal(8, cidr.PrefixLength);
Assert.Equal(IPAddress.Parse("255.0.0.0"), cidr.Mask);
cidr = new NetworkCidr(IPAddress.Parse("10.1.2.3"), 16);
Assert.Equal(IPAddress.Parse("10.1.0.0"), cidr.Address);
Assert.Equal(16, cidr.PrefixLength);
Assert.Equal(IPAddress.Parse("255.255.0.0"), cidr.Mask);
cidr = new NetworkCidr(IPAddress.Parse("10.1.2.3"), 24);
Assert.Equal(IPAddress.Parse("10.1.2.0"), cidr.Address);
Assert.Equal(24, cidr.PrefixLength);
Assert.Equal(IPAddress.Parse("255.255.255.0"), cidr.Mask);
}
public void Parse()
{
var cidr = NetworkCidr.Parse("10.1.2.3/8");
Assert.Equal(IPAddress.Parse("10.0.0.0"), cidr.Address);
Assert.True(NetHelper.AddressEquals(IPAddress.Parse("255.0.0.0"), cidr.Mask));
Assert.Equal(8, cidr.PrefixLength);
cidr = NetworkCidr.Parse("10.1.2.3/16");
Assert.Equal(IPAddress.Parse("10.1.0.0"), cidr.Address);
Assert.True(NetHelper.AddressEquals(IPAddress.Parse("255.255.0.0"), cidr.Mask));
Assert.Equal(16, cidr.PrefixLength);
cidr = NetworkCidr.Parse("10.1.2.3/24");
Assert.Equal(IPAddress.Parse("10.1.2.0"), cidr.Address);
Assert.True(NetHelper.AddressEquals(IPAddress.Parse("255.255.255.0"), cidr.Mask));
Assert.Equal(24, cidr.PrefixLength);
}
public void TryParse()
{
NetworkCidr cidr;
Assert.True(NetworkCidr.TryParse("10.1.2.3/8", out cidr));
Assert.Equal(IPAddress.Parse("10.0.0.0"), cidr.Address);
Assert.Equal(8, cidr.PrefixLength);
Assert.Equal(IPAddress.Parse("255.0.0.0"), cidr.Mask);
Assert.True(NetworkCidr.TryParse("10.1.2.3/16", out cidr));
Assert.Equal(IPAddress.Parse("10.1.0.0"), cidr.Address);
Assert.Equal(16, cidr.PrefixLength);
Assert.Equal(IPAddress.Parse("255.255.0.0"), cidr.Mask);
Assert.True(NetworkCidr.TryParse("10.1.2.3/24", out cidr));
Assert.Equal(IPAddress.Parse("10.1.2.0"), cidr.Address);
Assert.Equal(24, cidr.PrefixLength);
Assert.Equal(IPAddress.Parse("255.255.255.0"), cidr.Mask);
}
public void ValidatePrivateNodeAddresses()
{
var ipAddressToNode = new Dictionary <IPAddress, NodeDefinition>();
if (string.IsNullOrEmpty(Network.NodesSubnet))
{
throw new HiveDefinitionException($"The [{nameof(HiveDefinition)}.{nameof(HiveDefinition.Network)}.{nameof(NetworkOptions.NodesSubnet)}] property is required.");
}
if (!NetworkCidr.TryParse(Network.NodesSubnet, out var nodesSubnet))
{
throw new HiveDefinitionException($"The [{nameof(HiveDefinition)}.{nameof(HiveDefinition.Network)}.{nameof(NetworkOptions.NodesSubnet)}={Network.NodesSubnet}] property is not valid.");
}
foreach (var node in SortedNodes.OrderBy(n => n.Name))
{
if (string.IsNullOrEmpty(node.PrivateAddress))
{
throw new HiveDefinitionException($"Node [{node.Name}] has not been assigned a private IP address.");
}
if (!IPAddress.TryParse(node.PrivateAddress, out var address))
{
throw new HiveDefinitionException($"Node [{node.Name}] has invalid private IP address [{node.PrivateAddress}].");
}
if (address == IPAddress.Any)
{
throw new HiveDefinitionException($"Node [{node.Name}] has not been assigned a private IP address.");
}
if (ipAddressToNode.TryGetValue(address, out var conflictingNode))
{
throw new HiveDefinitionException($"Nodes [{conflictingNode.Name}] and [{node.Name}] have the same IP address [{address}].");
}
ipAddressToNode.Add(address, node);
}
}
/// <summary>
/// Validates the options and also ensures that all <c>null</c> properties are
/// initialized to their default values.
/// </summary>
/// <param name="clusterDefinition">The cluster definition.</param>
/// <exception cref="ClusterDefinitionException">Thrown if the definition is not valid.</exception>
public void Validate(ClusterDefinition clusterDefinition)
{
Covenant.Requires <ArgumentNullException>(clusterDefinition != null, nameof(clusterDefinition));
var googleHostingOptionsPrefix = $"{nameof(ClusterDefinition.Hosting)}.{nameof(ClusterDefinition.Hosting.Google)}";
// Verify subnets
if (!NetworkCidr.TryParse(VnetSubnet, out var vnetSubnet))
{
throw new ClusterDefinitionException($"[{googleHostingOptionsPrefix}.{nameof(VnetSubnet)}={VnetSubnet}] is not a valid subnet.");
}
if (!NetworkCidr.TryParse(NodeSubnet, out var nodeSubnet))
{
throw new ClusterDefinitionException($"[{googleHostingOptionsPrefix}.{nameof(NodeSubnet)}={NodeSubnet}] is not a valid subnet.");
}
if (!vnetSubnet.Contains(nodeSubnet))
{
throw new ClusterDefinitionException($"[{googleHostingOptionsPrefix}.{nameof(NodeSubnet)}={NodeSubnet}] is contained within [{nameof(VnetSubnet)}={VnetSubnet}].");
}
}
public void Compare()
{
Assert.True(NetworkCidr.Parse("10.0.0.1/8") == NetworkCidr.Parse("10.0.0.1/8"));
Assert.True(NetworkCidr.Parse("10.0.0.1/8").Equals(NetworkCidr.Parse("10.0.0.1/8")));
Assert.True(NetworkCidr.Parse("10.0.0.1/8") == NetworkCidr.Parse("10.0.2.1/8"));
Assert.False(NetworkCidr.Parse("10.0.0.1/8") == NetworkCidr.Parse("10.0.0.1/16"));
Assert.False(NetworkCidr.Parse("10.0.0.1/8") != NetworkCidr.Parse("10.0.0.1/8"));
Assert.True(NetworkCidr.Parse("10.0.0.1/8") == NetworkCidr.Parse("10.0.2.1/8"));
Assert.True(NetworkCidr.Parse("10.0.0.1/8") != NetworkCidr.Parse("10.0.0.1/16"));
Assert.Equal(NetworkCidr.Parse("10.0.0.1/8").GetHashCode(), NetworkCidr.Parse("10.0.0.1/8").GetHashCode());
Assert.NotEqual(NetworkCidr.Parse("10.0.0.1/8").GetHashCode(), NetworkCidr.Parse("10.0.0.1/16").GetHashCode());
Assert.Equal(NetworkCidr.Parse("10.0.0.1/8").GetHashCode(), NetworkCidr.Parse("10.0.0.2/8").GetHashCode());
Assert.True((NetworkCidr)null == (NetworkCidr)null);
Assert.False((NetworkCidr)null != (NetworkCidr)null);
Assert.False(NetworkCidr.Parse("10.0.0.1/8") == (NetworkCidr)null);
Assert.False((NetworkCidr)null == NetworkCidr.Parse("10.0.0.1/8"));
Assert.True(NetworkCidr.Parse("10.0.0.1/8") != (NetworkCidr)null);
Assert.True((NetworkCidr)null != NetworkCidr.Parse("10.0.0.1/8"));
}
public void Validate(HiveDefinition hiveDefinition)
{
Covenant.Requires <ArgumentNullException>(hiveDefinition != null);
var subnets = new List <SubnetDefinition>();
if (!NetworkCidr.TryParse(PublicSubnet, out var cidr))
{
throw new HiveDefinitionException($"Invalid [{nameof(NetworkOptions)}.{nameof(PublicSubnet)}={PublicSubnet}].");
}
subnets.Add(new SubnetDefinition(nameof(PublicSubnet), cidr));
if (!NetworkCidr.TryParse(PrivateSubnet, out cidr))
{
throw new HiveDefinitionException($"Invalid [{nameof(NetworkOptions)}.{nameof(PrivateSubnet)}={PrivateSubnet}].");
}
subnets.Add(new SubnetDefinition(nameof(PrivateSubnet), cidr));
if (PublicSubnet == PrivateSubnet)
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(PublicSubnet)}] cannot be the same as [{nameof(PrivateSubnet)}] .");
}
if (Nameservers == null || Nameservers.Length == 0)
{
Nameservers = new string[] { "8.8.8.8", "8.8.4.4" };
}
foreach (var nameserver in Nameservers)
{
if (!IPAddress.TryParse(nameserver, out var address))
{
throw new HiveDefinitionException($"[{nameserver}] is not a valid [{nameof(NetworkOptions)}.{nameof(Nameservers)}] IP address.");
}
}
PdnsRecursorPackageUri = PdnsRecursorPackageUri ?? defaultPdnsRecursorPackagePackageUri;
if (!Uri.TryCreate(PdnsRecursorPackageUri, UriKind.Absolute, out var uri3))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(PdnsRecursorPackageUri)}={PdnsRecursorPackageUri}] is not a valid URI.");
}
if (hiveDefinition.Hosting.IsCloudProvider)
{
// Verify [CloudSubnet].
if (string.IsNullOrEmpty(CloudSubnet))
{
CloudSubnet = defaultCloudSubnet;
}
if (!NetworkCidr.TryParse(CloudSubnet, out var cloudSubnetCidr))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(CloudSubnet)}={CloudSubnet}] is not a valid IPv4 subnet.");
}
if (cloudSubnetCidr.PrefixLength != 21)
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(CloudSubnet)}={CloudSubnet}] prefix length is not valid. Only [/21] subnets are currently supported.");
}
// Compute [NodeSubnet] by splitting [HiveSubnet] in quarters and taking the
// first quarter.
NetworkCidr nodesSubnetCidr;
nodesSubnetCidr = new NetworkCidr(cloudSubnetCidr.Address, cloudSubnetCidr.PrefixLength + 2);
NodesSubnet = nodesSubnetCidr.ToString();
subnets.Add(new SubnetDefinition(nameof(NodesSubnet), nodesSubnetCidr));
// Ensure that the node subnet is big enough to allocate an
// IP address for each node.
if (hiveDefinition.Nodes.Count() > nodesSubnetCidr.AddressCount - 4)
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(NodesSubnet)}={NodesSubnet}] subnet not large enough for the [{hiveDefinition.Nodes.Count()}] node addresses.");
}
// Verify/Compute VPN properties.
if (hiveDefinition.Vpn.Enabled)
{
// Compute [CloudVpnSubnet] by taking the second quarter of [HiveSubnet].
NetworkCidr cloudVpnCidr;
cloudVpnCidr = new NetworkCidr(nodesSubnetCidr.NextAddress, cloudSubnetCidr.PrefixLength + 2);
CloudVpnSubnet = cloudVpnCidr.ToString();
// Compute [CloudVNetSubnet] by taking the first half of [CloudSubnet],
// which includes both [NodesSubnet] and [CloudVpnSubnet].
NetworkCidr cloudVNetSubnet;
cloudVNetSubnet = new NetworkCidr(cloudSubnetCidr.Address, cloudSubnetCidr.PrefixLength + 1);
CloudVNetSubnet = cloudVNetSubnet.ToString();
// Compute [VpnPoolSubnet] by taking the upper half of [HiveSubnet].
NetworkCidr vpnPoolCidr;
vpnPoolCidr = new NetworkCidr(cloudVpnCidr.NextAddress, 22);
VpnPoolSubnet = vpnPoolCidr.ToString();
subnets.Add(new SubnetDefinition(nameof(VpnPoolSubnet), vpnPoolCidr));
}
}
else
{
// Verify [PremiseSubnet].
if (!NetworkCidr.TryParse(PremiseSubnet, out var premiseCidr))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(PremiseSubnet)}={PremiseSubnet}] is not a valid IPv4 subnet.");
}
// Verify [Gateway]
if (string.IsNullOrEmpty(Gateway))
{
// Default to the first valid address of the hive nodes subnet
// if this isn't already set.
Gateway = premiseCidr.FirstUsableAddress.ToString();
}
if (!IPAddress.TryParse(Gateway, out var gateway) || gateway.AddressFamily != AddressFamily.InterNetwork)
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(Gateway)}={Gateway}] is not a valid IPv4 address.");
}
if (!premiseCidr.Contains(gateway))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(Gateway)}={Gateway}] address is not within the [{nameof(NetworkOptions)}.{nameof(NetworkOptions.NodesSubnet)}={NodesSubnet}] subnet.");
}
// Verify [Broadcast]
if (string.IsNullOrEmpty(Broadcast))
{
// Default to the first valid address of the hive nodes subnet
// if this isn't already set.
Broadcast = premiseCidr.LastAddress.ToString();
}
if (!IPAddress.TryParse(Broadcast, out var broadcast) || broadcast.AddressFamily != AddressFamily.InterNetwork)
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(Broadcast)}={Broadcast}] is not a valid IPv4 address.");
}
if (!premiseCidr.Contains(broadcast))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(Broadcast)}={Broadcast}] address is not within the [{nameof(NetworkOptions)}.{nameof(NetworkOptions.NodesSubnet)}={NodesSubnet}] subnet.");
}
// Verify [NodesSubnet].
if (!NetworkCidr.TryParse(NodesSubnet, out var nodesSubnetCidr))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(NodesSubnet)}={NodesSubnet}] is not a valid IPv4 subnet.");
}
if (!premiseCidr.Contains(nodesSubnetCidr))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(NodesSubnet)}={NodesSubnet}] is not within [{nameof(NetworkOptions)}.{nameof(PremiseSubnet)}={PremiseSubnet}].");
}
// Verify VPN properties for on-premise environments.
if (hiveDefinition.Vpn.Enabled)
{
if (hiveDefinition.Hosting.IsOnPremiseProvider)
{
if (string.IsNullOrEmpty(ManagerPublicAddress))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(ManagerPublicAddress)}] is required for on-premise deployments that enable VPN. Set the public IP address or FQDN of your hive router.");
}
}
// Verify [VpnPoolSubnet].
if (!NetworkCidr.TryParse(VpnPoolSubnet, out var vpnPoolCidr))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(VpnPoolSubnet)}={VpnPoolSubnet}] is not a valid subnet.");
}
if (vpnPoolCidr.PrefixLength > 23)
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(VpnPoolSubnet)}={VpnPoolSubnet}] is too small. The subnet prefix length cannot be longer than [23].");
}
subnets.Add(new SubnetDefinition(nameof(VpnPoolSubnet), vpnPoolCidr));
if (nodesSubnetCidr.Overlaps(vpnPoolCidr))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(NodesSubnet)}={NodesSubnet}] and [{nameof(VpnPoolSubnet)}={VpnPoolSubnet}] overlap.");
}
subnets.Add(new SubnetDefinition(nameof(NodesSubnet), nodesSubnetCidr));
if (!premiseCidr.Contains(vpnPoolCidr))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(VpnPoolSubnet)}={VpnPoolSubnet}] is not within [{nameof(NetworkOptions)}.{nameof(PremiseSubnet)}={PremiseSubnet}].");
}
}
}
// Verify that none of the major subnets conflict.
foreach (var subnet in subnets)
{
foreach (var subnetTest in subnets)
{
if (subnet == subnetTest)
{
continue; // Don't test against self.[
}
if (subnet.Cidr.Overlaps(subnetTest.Cidr))
{
throw new HiveDefinitionException($"[{subnet.Name}={subnet.Cidr}] and [{subnetTest.Name}={subnetTest.Cidr}] overlap.");
}
}
}
// Verify the [NetworkMTU] settings.
if (MTU < 256)
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{MTU}={MTU}] cannot be less than [256].");
}
// Verify the ingress network settings.
if (IngressMTU < 256)
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{IngressMTU}={IngressMTU}] cannot be less than [256].");
}
if (!NetworkCidr.TryParse(IngressSubnet, out var ingressSubnet))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(IngressSubnet)}={IngressSubnet}] is not a valid subnet.");
}
if (!IPAddress.TryParse(IngressGateway, out var ingressGateway) || ingressGateway.AddressFamily != AddressFamily.InterNetwork)
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(IPAddress)}={IngressGateway}] is not a valid IPv4 address.");
}
if (!ingressSubnet.Contains(ingressGateway))
{
throw new HiveDefinitionException($"[{nameof(NetworkOptions)}.{nameof(IPAddress)}={IngressGateway}] is within the [{nameof(IngressSubnet)}={IngressSubnet}].");
}
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="name">Subnet name.</param>
/// <param name="cidr">Subnet CIDR.</param>
public SubnetDefinition(string name, NetworkCidr cidr)
{
this.Name = $"{nameof(NetworkOptions)}.{name}";
this.Cidr = cidr;
}
public void Validate()
{
// Wire up the node label parents.
foreach (var node in NodeDefinitions.Values)
{
if (node.Labels != null)
{
node.Labels.Node = node;
}
}
// Validate the properties.
Provisioner = Provisioner ?? defaultProvisioner;
Kubernetes = Kubernetes ?? new KubernetesOptions();
Docker = Docker ?? new DockerOptions();
Ceph = Ceph ?? new CephOptions()
{
Enabled = false
};
Mon = Mon ?? new MonOptions()
{
Enabled = false
};
Prometheus = Prometheus ?? new PrometheusOptions()
{
Enabled = false
};
DrivePrefix = DrivePrefix ?? defaultDrivePrefix;
Setup = Setup ?? new SetupOptions();
Hosting = Hosting ?? new HostingOptions();
NodeOptions = NodeOptions ?? new NodeOptions();
Network = Network ?? new NetworkOptions();
Kubernetes.Validate(this);
Docker.Validate(this);
Ceph.Validate(this);
Mon.Validate(this);
Prometheus.Validate(this);
Setup.Validate(this);
Network.Validate(this);
Hosting.Validate(this);
NodeOptions.Validate(this);
Network.Validate(this);
new HostingManagerFactory().Validate(this);
if (TimeSources == null || TimeSources.Length == 0 || TimeSources.Count(ts => string.IsNullOrWhiteSpace(ts)) > 0)
{
TimeSources = new string[] { "pool.ntp.org" };
}
if (NodeDefinitions == null || NodeDefinitions.Count == 0)
{
throw new ClusterDefinitionException("At least one cluster node must be defined.");
}
foreach (var node in NodeDefinitions.Values)
{
node.Validate(this);
}
if (Name == null)
{
throw new ClusterDefinitionException($"The [{nameof(ClusterDefinition)}.{nameof(Name)}] property is required.");
}
if (!IsValidName(Name))
{
throw new ClusterDefinitionException($"The [{nameof(ClusterDefinition)}.{nameof(Name)}={Name}] property is not valid. Only letters, numbers, periods, dashes, and underscores are allowed.");
}
if (Datacenter == null)
{
throw new ClusterDefinitionException($"The [{nameof(ClusterDefinition)}.{nameof(Datacenter)}] property is required.");
}
if (!IsValidName(Datacenter))
{
throw new ClusterDefinitionException($"The [{nameof(ClusterDefinition)}.{nameof(Datacenter)}={Datacenter}] property is not valid. Only letters, numbers, periods, dashes, and underscores are allowed.");
}
var masterNodeCount = Masters.Count();
if (masterNodeCount == 0)
{
throw new ClusterDefinitionException("Clusters must have at least one master node.");
}
else if (masterNodeCount > 5)
{
throw new ClusterDefinitionException("Clusters may not have more than [5] master nodes.");
}
else if (!NeonHelper.IsOdd(masterNodeCount))
{
throw new ClusterDefinitionException($"[{masterNodeCount}] master nodes is not allowed. Only an off number of master nodes is allowed: [1, 3, or 5]");
}
if (!string.IsNullOrEmpty(PackageProxy))
{
// Ensure that this is set to zero or more network endpoints
// formatted like:
//
// HOSTNAME:PORT
// ADDRESS:PORT
foreach (var endpoint in PackageProxy.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries))
{
var fields = endpoint.Split(':');
if (!IPAddress.TryParse(fields[0], out var address) && !NetHelper.IsValidHost(fields[0]))
{
throw new ClusterDefinitionException($"Invalid IP address or HOSTNAME [{fields[0]}] in [{nameof(ClusterDefinition)}.{nameof(PackageProxy)}={PackageProxy}].");
}
if (!int.TryParse(fields[1], out var port) || !NetHelper.IsValidPort(port))
{
throw new ClusterDefinitionException($"Invalid port [{fields[1]}] in [{nameof(ClusterDefinition)}.{nameof(PackageProxy)}={PackageProxy}].");
}
}
}
// Ensure that each node has a valid unique or NULL IP address.
NetworkCidr nodesSubnet = null;
if (Network.NodeSubnet != null)
{
nodesSubnet = NetworkCidr.Parse(Network.NodeSubnet);
}
var addressToNode = new Dictionary <string, NodeDefinition>();
foreach (var node in SortedNodes)
{
if (node.PrivateAddress != null)
{
NodeDefinition conflictNode;
if (addressToNode.TryGetValue(node.PrivateAddress, out conflictNode))
{
throw new ClusterDefinitionException($"Node [name={node.Name}] has invalid private IP address [{node.PrivateAddress}] that conflicts with node [name={conflictNode.Name}].");
}
}
}
foreach (var node in SortedNodes)
{
if (node.PrivateAddress != null)
{
if (!IPAddress.TryParse(node.PrivateAddress, out var address))
{
throw new ClusterDefinitionException($"Node [name={node.Name}] has invalid private IP address [{node.PrivateAddress}].");
}
if (nodesSubnet != null && !nodesSubnet.Contains(address))
{
throw new ClusterDefinitionException($"Node [name={node.Name}] has private IP address [{node.PrivateAddress}] that is not within the hosting [{nameof(Network.NodeSubnet)}={Network.NodeSubnet}].");
}
}
else if (!Hosting.IsCloudProvider)
{
throw new ClusterDefinitionException($"Node [name={node.Name}] is not assigned a private IP address. This is required when deploying to a [{nameof(Environment)}={Environment}] hosting environment.");
}
}
}
/// <summary>
/// Default constructor.
/// </summary>
public HyperVHostingOptions()
{
NeonKubeInternalSubnet = NetworkCidr.Parse("100.64.0.0/24");
NeonKubeInternalGateway = NeonKubeInternalSubnet.FirstUsableAddress; // 100.64.0.1
NeonDesktopNodeAddress = NetHelper.AddressIncrement(NeonKubeInternalSubnet.LastAddress, incrementBy: -1); // 100.64.0.254
}
public void Normalize()
{
Assert.Equal("10.0.0.0/14", NetworkCidr.Normalize(NetworkCidr.Parse("10.0.0.0/14").ToString()));
Assert.Equal("10.168.0.0/14", NetworkCidr.Normalize(NetworkCidr.Parse("10.170.0.0/14").ToString()));
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="address">The associated IP address.</param>
/// <param name="subnet">The network subnet.</param>
/// <param name="interfaceName">
/// Identifies the network interface or switch to which this address
/// is connected.
/// </param>
public GrpcVirtualIPAddress(string address, NetworkCidr subnet, string interfaceName)
{
this.Address = address;
this.Subnet = subnet.ToString();
this.InterfaceName = interfaceName;
}
public void Validate()
{
Provisioner = Provisioner ?? defaultProvisioner;
DrivePrefix = DrivePrefix ?? defaultDrivePrefix;
Setup = Setup ?? new SetupOptions();
Hosting = Hosting ?? new HostingOptions();
Vpn = Vpn ?? new VpnOptions();
HiveNode = HiveNode ?? new HiveNodeOptions();
Docker = Docker ?? new DockerOptions();
Image = Image ?? new ImageOptions();
Network = Network ?? new NetworkOptions();
Consul = Consul ?? new ConsulOptions();
Vault = Vault ?? new VaultOptions();
Log = Log ?? new LogOptions();
Dashboard = Dashboard ?? new DashboardOptions();
HiveFS = HiveFS ?? new HiveFSOptions();
Proxy = Proxy ?? new ProxyOptions();
HiveMQ = HiveMQ ?? new HiveMQOptions();
Setup.Validate(this);
Network.Validate(this);
Hosting.Validate(this);
Vpn.Validate(this);
HiveNode.Validate(this);
Docker.Validate(this);
Image.Validate(this);
Consul.Validate(this);
Vault.Validate(this);
Log.Validate(this);
Dashboard.Validate(this);
HiveFS.Validate(this);
Proxy.Validate(this);
HiveMQ.Validate(this);
new HostingManagerFactory().Validate(this);
if (TimeSources == null || TimeSources.Length == 0 || TimeSources.Count(ts => string.IsNullOrWhiteSpace(ts)) > 0)
{
TimeSources = new string[] { "pool.ntp.org" };
}
if (NodeDefinitions == null || NodeDefinitions.Count == 0)
{
throw new HiveDefinitionException("At least one hive node must be defined.");
}
foreach (var node in NodeDefinitions.Values)
{
node.Validate(this);
}
if (Name == null)
{
throw new HiveDefinitionException($"The [{nameof(HiveDefinition)}.{nameof(Name)}] property is required.");
}
if (!IsValidName(Name))
{
throw new HiveDefinitionException($"The [{nameof(HiveDefinition)}.{nameof(Name)}={Name}] property is not valid. Only letters, numbers, periods, dashes, and underscores are allowed.");
}
if (Datacenter == null)
{
throw new HiveDefinitionException($"The [{nameof(HiveDefinition)}.{nameof(Datacenter)}] property is required.");
}
if (!IsValidName(Datacenter))
{
throw new HiveDefinitionException($"The [{nameof(HiveDefinition)}.{nameof(Datacenter)}={Datacenter}] property is not valid. Only letters, numbers, periods, dashes, and underscores are allowed.");
}
if (!string.IsNullOrEmpty(PackageProxy))
{
var packageCacheUris = PackageProxy.Split(',');
for (int i = 0; i < packageCacheUris.Length; i++)
{
packageCacheUris[i] = packageCacheUris[i].Trim();
if (!Uri.TryCreate(packageCacheUris[i], UriKind.Absolute, out var aptProxyUri))
{
throw new HiveDefinitionException($"The [{nameof(HiveDefinition)}.{nameof(PackageProxy)}={PackageProxy}] includes [{packageCacheUris[i]}] which is not a valid URI.");
}
if (aptProxyUri.Scheme != "http")
{
throw new HiveDefinitionException($"The [{nameof(HiveDefinition)}.{nameof(PackageProxy)}={PackageProxy}] includes [{packageCacheUris[i]}] which does not have the [http] scheme.");
}
}
}
var managementNodeCount = Managers.Count();
if (managementNodeCount == 0)
{
throw new HiveDefinitionException("Hives must have at least one management node.");
}
else if (managementNodeCount > 5)
{
throw new HiveDefinitionException("Hives may not have more than [5] management nodes.");
}
else if (!NeonHelper.IsOdd(managementNodeCount))
{
throw new HiveDefinitionException("Hives must have an odd number of management nodes: [1, 3, or 5]");
}
// Ensure that each node has a valid unique or NULL IP address.
NetworkCidr nodesSubnet = null;
NetworkCidr vpnPoolSubnet = null;
if (Network.NodesSubnet != null)
{
nodesSubnet = NetworkCidr.Parse(Network.NodesSubnet);
}
if (Vpn.Enabled)
{
vpnPoolSubnet = NetworkCidr.Parse(Network.VpnPoolSubnet);
}
var addressToNode = new Dictionary <string, NodeDefinition>();
foreach (var node in SortedNodes)
{
if (node.PrivateAddress != null)
{
NodeDefinition conflictNode;
if (addressToNode.TryGetValue(node.PrivateAddress, out conflictNode))
{
throw new HiveDefinitionException($"Node [name={node.Name}] has invalid private IP address [{node.PrivateAddress}] that conflicts with node [name={conflictNode.Name}].");
}
}
}
foreach (var node in SortedNodes)
{
if (node.PrivateAddress != null)
{
if (!IPAddress.TryParse(node.PrivateAddress, out var address))
{
throw new HiveDefinitionException($"Node [name={node.Name}] has invalid private IP address [{node.PrivateAddress}].");
}
if (vpnPoolSubnet != null && vpnPoolSubnet.Contains(address))
{
throw new HiveDefinitionException($"Node [name={node.Name}] has private IP address [{node.PrivateAddress}] within the hosting [{nameof(Network.VpnPoolSubnet)}={Network.VpnPoolSubnet}].");
}
if (nodesSubnet != null && !nodesSubnet.Contains(address))
{
throw new HiveDefinitionException($"Node [name={node.Name}] has private IP address [{node.PrivateAddress}] that is not within the hosting [{nameof(Network.NodesSubnet)}={Network.NodesSubnet}].");
}
}
else if (!Hosting.IsCloudProvider)
{
throw new HiveDefinitionException($"Node [name={node.Name}] is not assigned a private IP address. This is required when deploying to a [{nameof(Environment)}={Environment}] hosting environment.");
}
}
// Verify that we have nodes identified for persisting log data if logging is enabled.
if (Log.Enabled)
{
if (Nodes.Where(n => n.Labels.LogEsData).Count() == 0)
{
throw new HiveDefinitionException($"At least one node must be configured to store log data by setting [{nameof(NodeDefinition.Labels)}.{nameof(NodeLabels.LogEsData)}=true] when hive logging is enabled.");
}
}
}
/// <summary>
/// Provision the virtual machines on the XenServer.
/// </summary>
/// <param name="xenSshProxy">The XenServer SSH proxy.</param>
private void ProvisionVirtualMachines(SshProxy <XenClient> xenSshProxy)
{
var xenHost = xenSshProxy.Metadata;
foreach (var node in GetHostedNodes(xenHost))
{
var vmName = GetVmName(node);
var processors = node.Metadata.GetVmProcessors(hive.Definition);
var memoryBytes = node.Metadata.GetVmMemory(hive.Definition);
var diskBytes = node.Metadata.GetVmDisk(hive.Definition);
xenSshProxy.Status = FormatVmStatus(vmName, "create virtual machine");
// We need to create a raw drive if the node hosts a Ceph OSD.
var extraDrives = new List <XenVirtualDrive>();
if (node.Metadata.Labels.CephOSD)
{
extraDrives.Add(
new XenVirtualDrive()
{
Size = node.Metadata.GetCephOSDDriveSize(hive.Definition)
});
}
var vm = xenHost.Machine.Create(vmName, hive.Definition.Hosting.XenServer.TemplateName,
processors: processors,
memoryBytes: memoryBytes,
diskBytes: diskBytes,
snapshot: hive.Definition.Hosting.XenServer.Snapshot,
extraDrives: extraDrives,
primaryStorageRepository: hive.Definition.Hosting.XenServer.StorageRepository,
extraStorageRespository: hive.Definition.Hosting.XenServer.OsdStorageRepository);
xenSshProxy.Status = FormatVmStatus(vmName, "start virtual machine");
xenHost.Machine.Start(vm);
// We need to wait for the virtual machine to start and obtain
// and IP address via DHCP.
var address = string.Empty;
xenSshProxy.Status = FormatVmStatus(vmName, "fetch ip address");
try
{
NeonHelper.WaitFor(
() =>
{
while (true)
{
vm = xenHost.Machine.Find(vmName);
if (!string.IsNullOrEmpty(vm.Address))
{
address = vm.Address;
return(true);
}
Thread.Sleep(1000);
}
},
TimeSpan.FromSeconds(120));
}
catch (TimeoutException)
{
xenSshProxy.Fault("Timeout waiting for virtual machine to start and set an IP address.");
}
// SSH into the VM using the DHCP address, configure the static IP
// address and extend the primary partition and file system to fill
// the drive and then reboot.
var subnet = NetworkCidr.Parse(hive.Definition.Network.PremiseSubnet);
var gateway = hive.Definition.Network.Gateway;
var broadcast = hive.Definition.Network.Broadcast;
// We're going to temporarily set the node to the current VM address
// so we can connect via SSH.
var savedNodeAddress = node.PrivateAddress;
try
{
node.PrivateAddress = IPAddress.Parse(address);
using (var nodeProxy = hive.GetNode(node.Name))
{
xenSshProxy.Status = FormatVmStatus(vmName, "connect");
nodeProxy.WaitForBoot();
// Replace the [/etc/network/interfaces] file to configure the static
// IP and then reboot to reinitialize networking subsystem.
var primaryInterface = node.GetNetworkInterface(node.PrivateAddress);
xenSshProxy.Status = FormatVmStatus(vmName, $"set static ip [{node.PrivateAddress}]");
var interfacesText =
$@"# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).
source /etc/network/interfaces.d/*
# The loopback network interface
auto lo
iface lo inet loopback
# The primary network interface
auto {primaryInterface}
iface {primaryInterface} inet static
address {savedNodeAddress}
netmask {subnet.Mask}
gateway {gateway}
broadcast {broadcast}
";
nodeProxy.UploadText("/etc/network/interfaces", interfacesText);
// Temporarily configure the public Google DNS servers as
// the name servers so DNS will work after we reboot with
// the static IP. Note that hive setup will eventually
// configure the name servers specified in the hive
// definition.
// $todo(jeff.lill):
//
// Is there a good reason why we're not just configuring the
// DNS servers from the hive definition here???
//
// Using the Google DNS seems like it could break some hive
// network configurations (e.g. for hives that don't have
// access to the public Internet). Totally private hives
// aren't really a supported scenario right now though because
// we assume we can use [apt-get]... to pull down packages.
var resolvBaseText =
$@"nameserver 8.8.8.8
nameserver 8.8.4.4
";
nodeProxy.UploadText("/etc/resolvconf/resolv.conf.d/base", resolvBaseText);
// Extend the primary partition and file system to fill
// the virtual the drive.
xenSshProxy.Status = FormatVmStatus(vmName, $"resize primary partition");
// $hack(jeff.lill):
//
// I've seen a transient error here but can't reproduce it. I'm going
// to assume for now that the file system might not be quite ready for
// this operation directly after the VM has been rebooted, so we're going
// to delay for a few seconds before performing the operations.
Thread.Sleep(TimeSpan.FromSeconds(5));
nodeProxy.SudoCommand("growpart /dev/xvda 1");
nodeProxy.SudoCommand("resize2fs /dev/xvda1");
// Reboot to pick up the changes.
xenSshProxy.Status = FormatVmStatus(vmName, "reboot");
nodeProxy.Reboot(wait: false);
}
}
finally
{
// Restore the node's IP address.
node.PrivateAddress = savedNodeAddress;
}
}
}
/// <summary>
/// Creates a Hyper-V virtual machine for a hive node.
/// </summary>
/// <param name="node">The target node.</param>
private void ProvisionVM(SshProxy <NodeDefinition> node)
{
// $todo(jeff.lill):
//
// This code currently assumes that the VM will use DHCP to obtain
// its initial network configuration so the code can SSH into the
// node to configure a static IP.
//
// It appears that it is possible to inject an IP address, but
// I wasn't able to get this to work (perhaps Windows Server is
// required. Here's a link discussing this:
//
// http://www.itprotoday.com/virtualization/modify-ip-configuration-vm-hyper-v-host
//
// An alternative technique might be to update [/etc/network/interfaces]
// remotely via PowerShell as described here:
//
// https://www.altaro.com/hyper-v/transfer-files-linux-hyper-v-guest/
using (var hyperv = new HyperVClient())
{
var vmName = GetVmName(node.Metadata);
// Extract the template file contents to the virtual machine's
// virtual hard drive file.
var drivePath = Path.Combine(vmDriveFolder, $"{vmName}-[0].vhdx");
using (var zip = new ZipFile(driveTemplatePath))
{
if (zip.Count != 1)
{
throw new ArgumentException($"[{driveTemplatePath}] ZIP archive includes more than one file.");
}
ZipEntry entry = null;
foreach (ZipEntry item in zip)
{
entry = item;
break;
}
if (!entry.IsFile)
{
throw new ArgumentException($"[{driveTemplatePath}] ZIP archive includes entry [{entry.Name}] that is not a file.");
}
if (!entry.Name.EndsWith(".vhdx", StringComparison.InvariantCultureIgnoreCase))
{
throw new ArgumentException($"[{driveTemplatePath}] ZIP archive includes a file that's not named like [*.vhdx].");
}
node.Status = $"create disk";
// $hack(jeff.lill): Update console at 2 sec intervals to avoid annoying flicker
var updateInterval = TimeSpan.FromSeconds(2);
var stopwatch = new Stopwatch();
stopwatch.Start();
using (var input = zip.GetInputStream(entry))
{
using (var output = new FileStream(drivePath, FileMode.Create, FileAccess.ReadWrite))
{
var buffer = new byte[64 * 1024];
int cb;
while (true)
{
cb = input.Read(buffer, 0, buffer.Length);
if (cb == 0)
{
break;
}
output.Write(buffer, 0, cb);
var percentComplete = (int)(((double)output.Length / (double)entry.Size) * 100.0);
if (stopwatch.Elapsed >= updateInterval || percentComplete >= 100.0)
{
node.Status = $"[{percentComplete}%] create disk";
stopwatch.Restart();
}
}
}
}
}
// Stop and delete the virtual machine if one exists.
if (hyperv.VMExists(vmName))
{
hyperv.StopVM(vmName);
hyperv.RemoveVM(vmName);
}
// We need to create a raw drive if the node hosts a Ceph OSD.
var extraDrives = new List <VirtualDrive>();
if (node.Metadata.Labels.CephOSD)
{
extraDrives.Add(
new VirtualDrive()
{
IsDynamic = true,
Size = node.Metadata.GetCephOSDDriveSize(hive.Definition),
Path = Path.Combine(vmDriveFolder, $"{vmName}-[1].vhdx")
});
}
// Create the virtual machine if it doesn't already exist.
var processors = node.Metadata.GetVmProcessors(hive.Definition);
var memoryBytes = node.Metadata.GetVmMemory(hive.Definition);
var minMemoryBytes = node.Metadata.GetVmMinimumMemory(hive.Definition);
var diskBytes = node.Metadata.GetVmDisk(hive.Definition);
node.Status = $"create virtual machine";
hyperv.AddVM(
vmName,
processorCount: processors,
diskSize: diskBytes.ToString(),
memorySize: memoryBytes.ToString(),
minimumMemorySize: minMemoryBytes.ToString(),
drivePath: drivePath,
switchName: switchName,
extraDrives: extraDrives);
node.Status = $"start virtual machine";
hyperv.StartVM(vmName);
// Retrieve the virtual machine's network adapters (there should only be one)
// to obtain the IP address we'll use to SSH into the machine and configure
// it's static IP.
node.Status = $"fetch ip address";
var adapters = hyperv.ListVMNetworkAdapters(vmName, waitForAddresses: true);
var adapter = adapters.FirstOrDefault();
var address = adapter.Addresses.First();
var subnet = NetworkCidr.Parse(hive.Definition.Network.PremiseSubnet);
var gateway = hive.Definition.Network.Gateway;
var broadcast = hive.Definition.Network.Broadcast;
if (adapter == null)
{
throw new HyperVException($"Virtual machine [{vmName}] has no network adapters.");
}
// We're going to temporarily set the node to the current VM address
// so we can connect via SSH.
var savedNodeAddress = node.PrivateAddress;
try
{
node.PrivateAddress = address;
using (var nodeProxy = hive.GetNode(node.Name))
{
node.Status = $"connecting";
nodeProxy.WaitForBoot();
// We need to ensure that the host folders exist.
nodeProxy.CreateHiveHostFolders();
// Replace the [/etc/network/interfaces] file to configure the static
// IP and then reboot to reinitialize networking subsystem.
var primaryInterface = node.GetNetworkInterface(address);
node.Status = $"set static ip [{savedNodeAddress}]";
var interfacesText =
$@"# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).
source /etc/network/interfaces.d/*
# The loopback network interface
auto lo
iface lo inet loopback
# The primary network interface
auto {primaryInterface}
iface {primaryInterface} inet static
address {savedNodeAddress}
netmask {subnet.Mask}
gateway {gateway}
broadcast {broadcast}
";
nodeProxy.UploadText("/etc/network/interfaces", interfacesText);
// Temporarily configure the public Google DNS servers as
// the name servers so DNS will work after we reboot with
// the static IP. Note that hive setup will eventually
// configure the name servers specified in the hive
// definition.
// $todo(jeff.lill):
//
// Is there a good reason why we're not just configuring the
// DNS servers from the hive definition here???
//
// Using the Google DNS seems like it could break some hive
// network configurations (e.g. for hives that don't have
// access to the public Internet). Totally private hives
// aren't really a supported scenario right now though because
// we assume we can use [apt-get]... to pull down packages.
var resolvBaseText =
$@"nameserver 8.8.8.8
nameserver 8.8.4.4
";
nodeProxy.UploadText("/etc/resolvconf/resolv.conf.d/base", resolvBaseText);
// Extend the primary partition and file system to fill
// the virtual the drive.
node.Status = $"resize primary partition";
// $hack(jeff.lill):
//
// I've seen a transient error here but can't reproduce it. I'm going
// to assume for now that the file system might not be quite ready for
// this operation directly after the VM has been rebooted, so we're going
// to delay for a few seconds before performing the operations.
Thread.Sleep(TimeSpan.FromSeconds(5));
nodeProxy.SudoCommand("growpart /dev/sda 1");
nodeProxy.SudoCommand("resize2fs /dev/sda1");
// Reboot to pick up the changes.
node.Status = $"rebooting";
nodeProxy.Reboot(wait: false);
}
}
finally
{
// Restore the node's IP address.
node.PrivateAddress = savedNodeAddress;
}
}
}
